Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

improvement: add file limit, warning, and more performant file detection (#7368)

Myles Scolnick4062b0fd585e
diff --git a/frontend/src/components/pages/home-page.tsx b/frontend/src/components/pages/home-page.tsx
index 6d963b3ae8..83dad963fc 100644
--- a/frontend/src/components/pages/home-page.tsx
+++ b/frontend/src/components/pages/home-page.tsx
@@ -169,6 +169,12 @@ const WorkspaceNotebooks: React.FC = () => {
return (
<WorkspaceRootContext value={workspace.root}>
<div className="flex flex-col gap-2">
+ {workspace.hasMore && (
+ <Banner kind="warn" className="rounded p-4">
+ Showing first {workspace.fileCount} files. Your workspace has more
+ files.
+ </Banner>
+ )}
<Header
Icon={BookTextIcon}
control={
diff --git a/marimo/_server/api/endpoints/home.py b/marimo/_server/api/endpoints/home.py
index b3a099a237..29033bb1a0 100644
--- a/marimo/_server/api/endpoints/home.py
+++ b/marimo/_server/api/endpoints/home.py
@@ -1,6 +1,7 @@
# Copyright 2024 Marimo. All rights reserved.
from __future__ import annotations
+import asyncio
import os
import tempfile
from typing import TYPE_CHECKING
@@ -11,7 +12,11 @@ from starlette.responses import JSONResponse
from marimo import _loggers
from marimo._server.api.deps import AppState
from marimo._server.api.utils import parse_request
-from marimo._server.file_router import LazyListOfFilesAppFileRouter
+from marimo._server.file_router import (
+ MAX_FILES,
+ LazyListOfFilesAppFileRouter,
+ count_files,
+)
from marimo._server.model import ConnectionState
from marimo._server.models.home import (
MarimoFile,
@@ -89,8 +94,18 @@ async def workspace_files(
)
root = session_manager.file_router.directory
- files = session_manager.file_router.files
- return WorkspaceFilesResponse(files=files, root=root)
+ # Run file scanning in thread pool to avoid blocking the server
+ files = await asyncio.to_thread(lambda: session_manager.file_router.files)
+
+ file_count = count_files(files)
+ has_more = file_count >= MAX_FILES
+
+ return WorkspaceFilesResponse(
+ files=files,
+ root=root,
+ has_more=has_more,
+ file_count=file_count,
+ )
def _get_active_sessions(app_state: AppState) -> list[MarimoFile]:
diff --git a/marimo/_server/file_router.py b/marimo/_server/file_router.py
index f3d64b7fcc..2598f2a618 100644
--- a/marimo/_server/file_router.py
+++ b/marimo/_server/file_router.py
@@ -23,6 +23,9 @@ if TYPE_CHECKING:
LOGGER = _loggers.marimo_logger()
+# Maximum number of files to return in workspace scan
+MAX_FILES = 1000
+
# Some unique identifier for a file
MarimoFileKey = str
@@ -260,14 +263,31 @@ class LazyListOfFilesAppFileRouter(AppFileRouter):
@property
def files(self) -> list[FileInfo]:
if self._lazy_files is None:
- self._lazy_files = self._load_files()
+ try:
+ self._lazy_files = self._load_files()
+ except HTTPException as e:
+ if e.status_code == HTTPStatus.REQUEST_TIMEOUT:
+ # Return partial results on timeout
+ LOGGER.warning(
+ "Timeout during file scan, returning partial results"
+ )
+ # self._lazy_files will already contain partial results
+ # set in _load_files before the exception was raised
+ if self._lazy_files is None:
+ self._lazy_files = []
+ else:
+ raise
return self._lazy_files
def _load_files(self) -> list[FileInfo]:
import time
start_time = time.time()
- MAX_EXECUTION_TIME = 5 # 5 seconds timeout
+ MAX_EXECUTION_TIME = 10 # 10 seconds timeout
+ file_count = [0] # Use list for closure mutability
+ accumulated_results: list[
+ FileInfo
+ ] = [] # For partial results on timeout
def recurse(
directory: str, depth: int = 0
@@ -275,10 +295,17 @@ class LazyListOfFilesAppFileRouter(AppFileRouter):
if depth > MAX_DEPTH:
return None
+ # Check file limit
+ if file_count[0] >= MAX_FILES:
+ LOGGER.warning(f"Reached maximum file limit ({MAX_FILES})")
+ return None
+
if time.time() - start_time > MAX_EXECUTION_TIME:
+ # Store accumulated results before raising timeout
+ self._lazy_files = accumulated_results
raise HTTPException(
status_code=HTTPStatus.REQUEST_TIMEOUT,
- detail="Request timed out: Loading workspace files took too long.", # noqa: E501
+ detail=f"Request timed out: Loading workspace files took too long. Showing first {file_count[0]} files.", # noqa: E501
)
try:
@@ -310,18 +337,23 @@ class LazyListOfFilesAppFileRouter(AppFileRouter):
children=children,
)
)
- elif entry.name.endswith(tuple(allowed_extensions)):
- if self._is_marimo_app(entry.path):
- files.append(
- FileInfo(
- id=entry.path,
- path=entry.path,
- name=entry.name,
- is_directory=False,
- is_marimo_file=True,
- last_modified=entry.stat().st_mtime,
- )
+ elif entry.name.endswith(allowed_extensions):
+ if is_marimo_app(entry.path):
+ file_count[0] += 1
+ file_info = FileInfo(
+ id=entry.path,
+ path=entry.path,
+ name=entry.name,
+ is_directory=False,
+ is_marimo_file=True,
+ last_modified=entry.stat().st_mtime,
)
+ files.append(file_info)
+ # Also add to accumulated results for partial loading
+ accumulated_results.append(file_info)
+ # Check if we've reached the limit
+ if file_count[0] >= MAX_FILES:
+ break
# Sort folders then files, based on natural sort (alpha, then num)
return sorted(folders, key=natural_sort_file) + sorted(
@@ -330,11 +362,26 @@ class LazyListOfFilesAppFileRouter(AppFileRouter):
MAX_DEPTH = 5
skip_dirs = {
+ # Python virtual environments
"venv",
+ ".venv",
+ ".virtualenv",
+ "__pypackages__",
+ # Python cache and build
"__pycache__",
+ "build",
+ "dist",
+ "eggs",
+ # Package management
"node_modules",
"site-packages",
- "eggs",
+ # Testing and tooling
+ ".tox",
+ ".nox",
+ ".pytest_cache",
+ ".mypy_cache",
+ # Version control
+ ".git",
}
allowed_extensions = (
(".py", ".md", ".qmd") if self.include_markdown else (".py",)
@@ -342,19 +389,6 @@ class LazyListOfFilesAppFileRouter(AppFileRouter):
return recurse(self.directory) or []
- def _is_marimo_app(self, full_path: str) -> bool:
- try:
- path = MarimoPath(full_path)
- contents = path.read_text()
- if path.is_markdown():
- return "marimo-version:" in contents
- if path.is_python():
- return "marimo.App" in contents and "import marimo" in contents
- return False
- except Exception as e:
- LOGGER.debug("Error reading file %s: %s", full_path, e)
- return False
-
def get_unique_file_key(self) -> str | None:
return None
@@ -480,3 +514,56 @@ def validate_inside_directory(directory: Path, filepath: Path) -> None:
status_code=HTTPStatus.SERVER_ERROR,
detail=f"Unexpected error validating path: {str(e)}",
) from e
+
+
+def is_marimo_app(full_path: str) -> bool:
+ """
+ Detect whether a file is a marimo app.
+
+ Rules:
+ - Markdown (`.md`/`.qmd`) files are marimo apps if the first 512 bytes
+ contain `marimo-version:`.
+ - Python (`.py`) files are marimo apps if the header (first 512 bytes)
+ contains both `marimo.App` and `import marimo`.
+ - If the header contains `# /// script`, read the full file and check for
+ the same Python markers, to handle large script headers.
+ - Any errors while reading result in `False`.
+ """
+ READ_LIMIT = 512
+
+ def contains_marimo_app(content: bytes) -> bool:
+ return b"marimo.App" in content and b"import marimo" in content
+
+ try:
+ path = MarimoPath(full_path)
+
+ with open(full_path, "rb") as f:
+ header = f.read(READ_LIMIT)
+
+ if path.is_markdown():
+ return b"marimo-version:" in header
+
+ if path.is_python():
+ if contains_marimo_app(header):
+ return True
+
+ if b"# /// script" in header:
+ full_content = path.read_bytes()
+ if contains_marimo_app(full_content):
+ return True
+
+ return False
+ except Exception as e:
+ LOGGER.debug("Error reading file %s: %s", full_path, e)
+ return False
+
+
+# Count total marimo files (not directories)
+def count_files(file_list: list[FileInfo]) -> int:
+ count = 0
+ for item in file_list:
+ if not item.is_directory:
+ count += 1
+ if item.children:
+ count += count_files(item.children)
+ return count
diff --git a/marimo/_server/models/home.py b/marimo/_server/models/home.py
index 6f440bf47e..7ec173b1cb 100644
--- a/marimo/_server/models/home.py
+++ b/marimo/_server/models/home.py
@@ -43,6 +43,10 @@ class WorkspaceFilesRequest(msgspec.Struct, rename="camel"):
class WorkspaceFilesResponse(msgspec.Struct, rename="camel"):
root: str
files: list[FileInfo]
+ # Indicates if limit was reached
+ has_more: bool = False
+ # Total files found
+ file_count: int = 0
class ShutdownSessionRequest(msgspec.Struct, rename="camel"):
diff --git a/marimo/_utils/marimo_path.py b/marimo/_utils/marimo_path.py
index bd39c10802..a6fdae6956 100644
--- a/marimo/_utils/marimo_path.py
+++ b/marimo/_utils/marimo_path.py
@@ -85,6 +85,9 @@ class MarimoPath:
def read_text(self, encoding: str = "utf-8") -> str:
return self.path.read_text(encoding)
+ def read_bytes(self) -> bytes:
+ return self.path.read_bytes()
+
@property
def short_name(self) -> str:
return self.path.name
diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yaml
index 1e765aabe0..58f17b4d4c 100644
--- a/packages/openapi/api.yaml
+++ b/packages/openapi/api.yaml
@@ -3692,10 +3692,16 @@ components:
type: object
WorkspaceFilesResponse:
properties:
+ fileCount:
+ default: 0
+ type: integer
files:
items:
$ref: '#/components/schemas/FileInfo'
type: array
+ hasMore:
+ default: false
+ type: boolean
root:
type: string
required:
@@ -3755,7 +3761,7 @@ components:
type: object
info:
title: marimo API
- version: 0.17.6
+ version: 0.18.1
openapi: 3.1.0
paths:
/@file/{filename_and_length}:
diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.ts
index ae91fd7b91..f2d0b86994 100644
--- a/packages/openapi/src/api.ts
+++ b/packages/openapi/src/api.ts
@@ -4750,7 +4750,11 @@ export interface components {
};
/** WorkspaceFilesResponse */
WorkspaceFilesResponse: {
+ /** @default 0 */
+ fileCount?: number;
files: components["schemas"]["FileInfo"][];
+ /** @default false */
+ hasMore?: boolean;
root: string;
};
/**
diff --git a/pixi.lock b/pixi.lock
index 06962b9302..ddc976285e 100644
--- a/pixi.lock
+++ b/pixi.lock
@@ -1101,8 +1101,8 @@ packages:
timestamp: 1727801725384
- pypi: ./
name: marimo
- version: 0.18.0
- sha256: 38a7a4e5f4bd2017f356d0959dc2b5f0049990f4e416e6bf9ea6d35eeb67c9e7
+ version: 0.18.1
+ sha256: b89cff1b4327d087653d929b56b08026b0b690464da595eb8bb4d5dbadc971e1
requires_dist:
- click>=8.0,<9
- jedi>=0.18.0
diff --git a/tests/_server/api/endpoints/test_home.py b/tests/_server/api/endpoints/test_home.py
index a9bf907bcf..d6c03024fb 100644
--- a/tests/_server/api/endpoints/test_home.py
+++ b/tests/_server/api/endpoints/test_home.py
@@ -35,6 +35,11 @@ def test_workspace_files(client: TestClient) -> None:
files = body["files"]
assert len(files) == 1
assert files[0]["path"] == current_filename
+ # Check that new fields are present
+ assert "hasMore" in body
+ assert "fileCount" in body
+ assert body["hasMore"] is False
+ assert body["fileCount"] == 1
@with_session(SESSION_ID)
diff --git a/tests/_server/test_file_router.py b/tests/_server/test_file_router.py
index 2529dfea73..4ce410c842 100644
--- a/tests/_server/test_file_router.py
+++ b/tests/_server/test_file_router.py
@@ -14,6 +14,8 @@ from marimo._server.file_router import (
LazyListOfFilesAppFileRouter,
ListOfFilesAppFileRouter,
NewFileAppFileRouter,
+ count_files,
+ is_marimo_app,
validate_inside_directory,
)
from marimo._server.models.home import MarimoFile
@@ -402,3 +404,133 @@ class TestValidateInsideDirectory(unittest.TestCase):
with pytest.raises(HTTPException) as exc_info:
validate_inside_directory(directory, filepath)
assert exc_info.value.status_code == HTTPStatus.FORBIDDEN
+
+
+def test_python_app_detected_in_header(tmp_path: Path):
+ f = tmp_path / "app.py"
+ content = b"import marimo\napp = marimo.App()\n"
+ f.write_bytes(content)
+ assert is_marimo_app(str(f)) is True
+
+
+def test_python_app_detected_with_script_header_full_read(tmp_path: Path):
+ f = tmp_path / "script_app.py"
+ header = b"# /// script\n# lots of stuff before markers\n" + (b"x" * 600)
+ # Ensure markers are only in full content beyond header limit
+ body = b"\nimport marimo\napp = marimo.App()\n"
+ f.write_bytes(header + body)
+ assert is_marimo_app(str(f)) is True
+
+
+def test_python_non_app_returns_false(tmp_path: Path):
+ f = tmp_path / "not_app.py"
+ f.write_bytes(b"print('hello')\n")
+ assert is_marimo_app(str(f)) is False
+
+
+def test_markdown_with_marimo_version_detected(tmp_path: Path):
+ f = tmp_path / "notebook.md"
+ # Place marker in the first 512 bytes
+ f.write_bytes(b"---\nmarimo-version: 0.1\n---\n")
+ assert is_marimo_app(str(f)) is True
+
+
+def test_markdown_without_marimo_version_returns_false(tmp_path: Path):
+ f = tmp_path / "plain.md"
+ f.write_bytes(b"# Title\nSome content\n")
+ assert is_marimo_app(str(f)) is False
+
+
+def test_error_path_returns_false_and_logs(tmp_path: Path):
+ # Point to a directory to trigger open error
+ d = tmp_path / "adir"
+ d.mkdir()
+ assert is_marimo_app(str(d)) is False
+
+
+def test_lazy_router_respects_max_files(tmp_path: Path):
+ """Test that LazyListOfFilesAppFileRouter enforces MAX_FILES limit"""
+ # Create a directory with more files than MAX_FILES
+ # To make this test fast, we'll use a monkey-patch approach
+ # by temporarily reducing MAX_FILES
+ import marimo._server.file_router as file_router_module
+
+ original_max_files = file_router_module.MAX_FILES
+ try:
+ # Set a small limit for testing
+ file_router_module.MAX_FILES = 5
+
+ # Create 10 marimo files
+ for i in range(10):
+ f = tmp_path / f"app_{i}.py"
+ f.write_text("import marimo\napp = marimo.App()\n")
+
+ router = LazyListOfFilesAppFileRouter(
+ str(tmp_path), include_markdown=False
+ )
+ files = router.files
+
+ # Should only get MAX_FILES worth of files
+ # Count actual marimo files (not directories)
+ file_count = sum(1 for f in files if not f.is_directory)
+ assert file_count <= 5
+
+ finally:
+ # Restore original value
+ file_router_module.MAX_FILES = original_max_files
+
+
+def test_lazy_router_skips_common_dirs(tmp_path: Path):
+ """Test that LazyListOfFilesAppFileRouter skips common directories"""
+ # Create directories that should be skipped
+ skip_dirs = [
+ ".venv",
+ ".git",
+ "__pycache__",
+ "node_modules",
+ ".tox",
+ ".pytest_cache",
+ ]
+
+ for skip_dir in skip_dirs:
+ dir_path = tmp_path / skip_dir
+ dir_path.mkdir()
+ # Create a marimo file inside
+ f = dir_path / "app.py"
+ f.write_text("import marimo\napp = marimo.App()\n")
+
+ # Create a valid marimo file in the root
+ root_file = tmp_path / "root_app.py"
+ root_file.write_text("import marimo\napp = marimo.App()\n")
+
+ router = LazyListOfFilesAppFileRouter(
+ str(tmp_path), include_markdown=False
+ )
+ files = router.files
+
+ # Should only find the root file, not files in skipped directories
+ file_paths = [f.path for f in files if not f.is_directory]
+ assert len(file_paths) == 1
+ assert str(root_file) in file_paths
+
+
+def test_lazy_router_counts_nested_files(tmp_path: Path):
+ """Test that file counting works correctly with nested directories"""
+ # Create nested structure
+ nested_dir = tmp_path / "subdir"
+ nested_dir.mkdir()
+
+ # Create files at different levels
+ root_file = tmp_path / "root.py"
+ root_file.write_text("import marimo\napp = marimo.App()\n")
+
+ nested_file = nested_dir / "nested.py"
+ nested_file.write_text("import marimo\napp = marimo.App()\n")
+
+ router = LazyListOfFilesAppFileRouter(
+ str(tmp_path), include_markdown=False
+ )
+ files = router.files
+
+ total_files = count_files(files)
+ assert total_files == 2
diff --git a/tests/_utils/test_marimo_path.py b/tests/_utils/test_marimo_path.py
index 6f768ae18d..38f04646f9 100644
--- a/tests/_utils/test_marimo_path.py
+++ b/tests/_utils/test_marimo_path.py
@@ -79,6 +79,20 @@ def test_write_text(tmp_path: Path):
assert tmp_file.read_text() == text
+def test_read_bytes(tmp_path: Path):
+ # Create a temporary file with binary content
+ tmp_file = tmp_path / "test.py"
+ binary_content = b"import marimo\napp = marimo.App()\n"
+ tmp_file.write_bytes(binary_content)
+
+ mp = MarimoPath(tmp_file)
+
+ # Read bytes and verify
+ content = mp.read_bytes()
+ assert content == binary_content
+ assert isinstance(content, bytes)
+
+
def test_properties():
mp = MarimoPath(__file__)