Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(home): surface when the workspace file scan is truncated (#10550)

Chandan P4af1174d4cd2
diff --git a/frontend/src/components/home/components.tsx b/frontend/src/components/home/components.tsx
index 7ad26d35a0..a852a5954f 100644
--- a/frontend/src/components/home/components.tsx
+++ b/frontend/src/components/home/components.tsx
@@ -31,6 +31,7 @@ import {
import { Constants } from "@/core/constants";
import { useRequestClient } from "@/core/network/requests";
import type { TutorialId } from "@/core/network/types";
+import { Banner } from "@/plugins/impl/common/error-banner";
import { openNotebook } from "@/utils/links";
import { Objects } from "@/utils/objects";
import { MarimoPlusIcon } from "../icons/marimo-icons";
@@ -186,6 +187,22 @@ export const ResourceLinks: React.FC = () => {
);
};
+/**
+ * Shown when the workspace scan stopped early, so the file list the user is
+ * looking at is missing files and folders.
+ */
+export const TruncatedWorkspaceBanner: React.FC<{
+ fileCount: number | undefined;
+}> = ({ fileCount }) => {
+ return (
+ <Banner kind="warn" className="rounded p-4">
+ Showing {fileCount} files. Some files and folders were skipped because
+ your workspace is large or deeply nested. Start marimo from a subfolder to
+ see them.
+ </Banner>
+ );
+};
+
export const Header: React.FC<{
Icon: React.FC<React.SVGProps<SVGSVGElement>>;
control?: React.ReactNode;
diff --git a/frontend/src/components/pages/gallery-page.tsx b/frontend/src/components/pages/gallery-page.tsx
index 1e8044f5a1..0cb7c81d3c 100644
--- a/frontend/src/components/pages/gallery-page.tsx
+++ b/frontend/src/components/pages/gallery-page.tsx
@@ -4,6 +4,7 @@ import { SearchIcon } from "lucide-react";
import type React from "react";
import { Suspense, useMemo, useState } from "react";
import { ErrorBoundary } from "@/components/editor/boundary/ErrorBoundary";
+import { TruncatedWorkspaceBanner } from "@/components/home/components";
import { Spinner } from "@/components/icons/spinner";
import { Card, CardContent } from "@/components/ui/card";
import { Input } from "@/components/ui/input";
@@ -119,10 +120,7 @@ const GalleryPage: React.FC = () => {
<ErrorBoundary>
<div className="flex flex-col gap-2">
{workspace.hasMore && (
- <Banner kind="warn" className="rounded p-4">
- Showing first {workspace.fileCount} files. Your workspace has
- more files.
- </Banner>
+ <TruncatedWorkspaceBanner fileCount={workspace.fileCount} />
)}
{formattedFiles.length > SEARCH_THRESHOLD && (
<Input
diff --git a/frontend/src/components/pages/home-page.tsx b/frontend/src/components/pages/home-page.tsx
index 763db0813e..aba169f0a3 100644
--- a/frontend/src/components/pages/home-page.tsx
+++ b/frontend/src/components/pages/home-page.tsx
@@ -67,6 +67,7 @@ import {
Header,
OpenTutorialDropDown,
ResourceLinks,
+ TruncatedWorkspaceBanner,
} from "../home/components";
import {
expandedFoldersAtom,
@@ -196,10 +197,7 @@ const WorkspaceNotebooks: React.FC<{ onRefreshRecents: () => void }> = ({
<WorkspaceContext value={workspaceContextValue}>
<div className="flex flex-col gap-2">
{workspace.hasMore && (
- <Banner kind="warn" className="rounded p-4">
- Showing first {workspace.fileCount} files. Your workspace has more
- files.
- </Banner>
+ <TruncatedWorkspaceBanner fileCount={workspace.fileCount} />
)}
<Header
Icon={BookTextIcon}
diff --git a/marimo/_server/api/endpoints/home.py b/marimo/_server/api/endpoints/home.py
index 1acee1bf93..d055634202 100644
--- a/marimo/_server/api/endpoints/home.py
+++ b/marimo/_server/api/endpoints/home.py
@@ -153,7 +153,9 @@ async def workspace_files(
marimo_files = await asyncio.to_thread(get_files_with_metadata)
file_count = len(marimo_files)
- has_more = file_count >= MAX_FILES
+ has_more = (
+ file_count >= MAX_FILES or session_manager.workspace.is_truncated
+ )
return WorkspaceFilesResponse(
files=marimo_files,
root=session_manager.workspace.directory or "",
@@ -171,7 +173,9 @@ async def workspace_files(
files = await asyncio.to_thread(lambda: session_manager.workspace.files)
file_count = count_files(files)
- has_more = file_count >= MAX_FILES
+ has_more = (
+ file_count >= MAX_FILES or session_manager.workspace.is_truncated
+ )
return WorkspaceFilesResponse(
files=files,
diff --git a/marimo/_server/files/directory_scanner.py b/marimo/_server/files/directory_scanner.py
index d8c60d45c1..4c8a03fe67 100644
--- a/marimo/_server/files/directory_scanner.py
+++ b/marimo/_server/files/directory_scanner.py
@@ -76,6 +76,7 @@ class DirectoryScanner:
- Skip common directories (venv, node_modules, etc.)
- File count limits and timeouts
- Marimo app detection
+ - Reports (via `truncated`) when a limit cut the walk short
"""
MAX_DEPTH = 5
@@ -138,6 +139,11 @@ class DirectoryScanner:
)
# Stores partial results in case of timeout
self.partial_results: list[FileInfo] = []
+ # Set when a limit (depth, file count, time) or an unreadable entry
+ # stopped the walk before the tree was fully explored. Lets callers
+ # tell "this directory has no notebooks" apart from "we stopped
+ # looking".
+ self.truncated = False
@property
def allowed_extensions(self) -> tuple[str, ...]:
@@ -159,9 +165,11 @@ class DirectoryScanner:
start_time = time.time()
file_count = [0] # Use list for closure mutability
self.partial_results = [] # Reset partial results
+ self.truncated = False # Reset truncation flag
def recurse(directory: str, depth: int = 0) -> list[FileInfo] | None:
if depth > self.max_depth:
+ self.truncated = True
return None
# Check file limit
@@ -169,10 +177,12 @@ class DirectoryScanner:
LOGGER.warning(
f"Reached maximum file limit ({self.max_files})"
)
+ self.truncated = True
return None
if time.time() - start_time > self.max_execution_time:
# Store accumulated results before raising timeout
+ self.truncated = True
raise HTTPException(
status_code=HTTPStatus.REQUEST_TIMEOUT,
detail=f"Request timed out: Loading workspace files took too long. Showing first {file_count[0]} files.",
@@ -182,6 +192,7 @@ class DirectoryScanner:
entries = os.scandir(directory)
except OSError as e:
LOGGER.debug("OSError scanning directory: %s", str(e))
+ self.truncated = True
return None
files: list[FileInfo] = []
@@ -191,6 +202,7 @@ class DirectoryScanner:
# Check the limit here, not just after adding a file: a
# sibling directory's recursion may have reached it.
if file_count[0] >= self.max_files:
+ self.truncated = True
break
# Skip hidden files and directories
@@ -206,9 +218,10 @@ class DirectoryScanner:
if (
entry.name in self.SKIP_DIRS
or entry.name.lower() in self.SKIP_DIRS
- or depth == self.max_depth
):
continue
+ # Past `max_depth` this returns `None` without any
+ # I/O, and records that the walk was cut short.
children = recurse(entry.path, depth + 1)
if children:
entry_path = Path(entry.path)
@@ -247,6 +260,9 @@ class DirectoryScanner:
LOGGER.debug(
"Error processing entry %s: %s", entry.path, e
)
+ # We don't know what this entry was, so it may have been a
+ # notebook or a folder holding notebooks.
+ self.truncated = True
continue
# Sort folders then files, based on natural sort (alpha, then num)
diff --git a/marimo/_server/models/home.py b/marimo/_server/models/home.py
index f0f1beca40..3aede5b2a7 100644
--- a/marimo/_server/models/home.py
+++ b/marimo/_server/models/home.py
@@ -41,7 +41,8 @@ class WorkspaceFilesRequest(msgspec.Struct, rename="camel"):
class WorkspaceFilesResponse(msgspec.Struct, rename="camel"):
root: str
files: list[FileInfo]
- # Indicates if limit was reached
+ # Indicates the listing is incomplete: a file count, depth, or time limit
+ # was reached while scanning, so some files or folders were left out
has_more: bool = False
# Total files found
file_count: int = 0
diff --git a/marimo/_server/workspace/_base.py b/marimo/_server/workspace/_base.py
index 148ae753c3..80adccb8a0 100644
--- a/marimo/_server/workspace/_base.py
+++ b/marimo/_server/workspace/_base.py
@@ -45,6 +45,16 @@ class NotebookWorkspace(abc.ABC):
def files(self) -> list[FileInfo]:
"""All files in this workspace as a recursive tree."""
+ @property
+ def is_truncated(self) -> bool:
+ """Whether the last listing was cut short and may be incomplete.
+
+ `False` by default — workspaces with a fixed set of notebooks always
+ list all of them. `DirectoryWorkspace` overrides, since its scan is
+ bounded by depth, file count, and time limits.
+ """
+ return False
+
@abc.abstractmethod
def single_file(self) -> MarimoFile | None:
"""If this workspace represents a single notebook, return it."""
diff --git a/marimo/_server/workspace/_directory.py b/marimo/_server/workspace/_directory.py
index 499ecdbe3d..568881435b 100644
--- a/marimo/_server/workspace/_directory.py
+++ b/marimo/_server/workspace/_directory.py
@@ -107,3 +107,14 @@ class DirectoryWorkspace(NotebookWorkspace):
else:
raise
return self._lazy_files
+
+ @property
+ def is_truncated(self) -> bool:
+ """Whether the scan hit a depth, file count, or time limit.
+
+ Notebooks nested deeper than the scanner's depth limit are dropped
+ along with their folders, which otherwise looks identical to an empty
+ directory; this lets the client say so instead of silently omitting
+ them.
+ """
+ return self._scanner.truncated
diff --git a/tests/_server/test_directory_scanner.py b/tests/_server/test_directory_scanner.py
index c0a9d4b28c..9e5619534c 100644
--- a/tests/_server/test_directory_scanner.py
+++ b/tests/_server/test_directory_scanner.py
@@ -206,6 +206,81 @@ class TestDirectoryScanner:
files = DirectoryScanner(str(test_dir)).scan()
assert "long_docstring_app.py" in _file_names(files)
+ def test_truncated_when_folders_are_deeper_than_max_depth(
+ self, tmp_path: Path
+ ):
+ """Notebooks past `max_depth` are dropped, but not silently (#10064).
+
+ The folder holding them looks exactly like an empty folder, so the
+ scanner has to report that it stopped looking.
+ """
+ deep = tmp_path / "projects" / "analysis"
+ deep.mkdir(parents=True)
+ _write(deep / "app.py", MARIMO_APP)
+
+ scanner = DirectoryScanner(str(tmp_path), max_depth=1)
+ files = scanner.scan()
+
+ assert _count_files(files) == 0
+ assert scanner.truncated
+
+ def test_not_truncated_when_tree_is_fully_scanned(self, test_dir: Path):
+ scanner = DirectoryScanner(str(test_dir))
+ scanner.scan()
+ assert not scanner.truncated
+
+ def test_truncated_when_max_files_reached(self, test_dir: Path):
+ for i in range(10):
+ _write(test_dir / f"app{i + 3}.py", MARIMO_APP)
+ scanner = DirectoryScanner(str(test_dir), max_files=5)
+ scanner.scan()
+ assert scanner.truncated
+
+ def test_truncated_is_reset_between_scans(self, tmp_path: Path):
+ deep = tmp_path / "projects" / "analysis"
+ deep.mkdir(parents=True)
+ _write(deep / "app.py", MARIMO_APP)
+
+ scanner = DirectoryScanner(str(tmp_path), max_depth=1)
+ scanner.scan()
+ assert scanner.truncated
+
+ # Deep enough to reach the notebook: no truncation this time.
+ scanner.max_depth = 2
+ files = scanner.scan()
+ assert _count_files(files) == 1
+ assert not scanner.truncated
+
+ def test_truncated_when_an_entry_cannot_be_read(
+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ):
+ """An entry we fail to stat may have been a notebook, or a folder
+ holding notebooks, so it counts as a truncated scan too."""
+ import marimo._server.files.directory_scanner as scanner_mod
+
+ (tmp_path / "projects").mkdir()
+ _write(tmp_path / "projects" / "app.py", MARIMO_APP)
+
+ class UnreadableEntry:
+ name = "projects"
+ path = str(tmp_path / "projects")
+
+ def is_symlink(self) -> bool:
+ return False
+
+ def is_dir(self) -> bool:
+ raise PermissionError("denied")
+
+ monkeypatch.setattr(
+ scanner_mod.os, "scandir", lambda _path: iter([UnreadableEntry()])
+ )
+
+ scanner = DirectoryScanner(str(tmp_path))
+ files = scanner.scan()
+
+ assert files == []
+ assert scanner.truncated
+
def test_partial_results_populated_during_scan(self, test_dir: Path):
scanner = DirectoryScanner(str(test_dir))
assert scanner.partial_results == []
diff --git a/tests/_server/test_workspace.py b/tests/_server/test_workspace.py
index 73d9401e0b..786354876b 100644
--- a/tests/_server/test_workspace.py
+++ b/tests/_server/test_workspace.py
@@ -750,6 +750,28 @@ def test_lazy_router_respects_max_files(tmp_path: Path):
DirectoryScanner.MAX_FILES = original_max_files
+def test_lazy_router_reports_truncated_scan(tmp_path: Path):
+ """Deeply nested notebooks are omitted, and the workspace says so (#10064)."""
+ deep = tmp_path.joinpath("a", "b", "c", "d", "e", "f")
+ deep.mkdir(parents=True)
+ (deep / "app.py").write_text(file_contents)
+
+ router = DirectoryWorkspace(str(tmp_path), include_markdown=False)
+
+ # The notebook sits past DirectoryScanner.MAX_DEPTH, so nothing is listed.
+ assert router.files == []
+ assert router.is_truncated
+
+
+def test_lazy_router_not_truncated_for_shallow_tree(tmp_path: Path):
+ (tmp_path / "app.py").write_text(file_contents)
+
+ router = DirectoryWorkspace(str(tmp_path), include_markdown=False)
+
+ assert len(router.files) == 1
+ assert not router.is_truncated
+
+
def test_lazy_router_skips_common_dirs(tmp_path: Path):
"""Test that DirectoryWorkspace skips common directories"""
# Create directories that should be skipped