A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(home): surface when the workspace file scan is truncated (#10550)
Chandan P4af1174d4cd2
diff --git a/frontend/src/components/home/components.tsx b/frontend/src/components/home/components.tsxindex 7ad26d35a0..a852a5954f 100644--- a/frontend/src/components/home/components.tsx+++ b/frontend/src/components/home/components.tsx@@ -31,6 +31,7 @@ import {import { Constants } from "@/core/constants";import { useRequestClient } from "@/core/network/requests";import type { TutorialId } from "@/core/network/types";+import { Banner } from "@/plugins/impl/common/error-banner";import { openNotebook } from "@/utils/links";import { Objects } from "@/utils/objects";import { MarimoPlusIcon } from "../icons/marimo-icons";@@ -186,6 +187,22 @@ export const ResourceLinks: React.FC = () => {);};+/**+ * Shown when the workspace scan stopped early, so the file list the user is+ * looking at is missing files and folders.+ */+export const TruncatedWorkspaceBanner: React.FC<{+ fileCount: number | undefined;+}> = ({ fileCount }) => {+ return (+ <Banner kind="warn" className="rounded p-4">+ Showing {fileCount} files. Some files and folders were skipped because+ your workspace is large or deeply nested. Start marimo from a subfolder to+ see them.+ </Banner>+ );+};+export const Header: React.FC<{Icon: React.FC<React.SVGProps<SVGSVGElement>>;control?: React.ReactNode;diff --git a/frontend/src/components/pages/gallery-page.tsx b/frontend/src/components/pages/gallery-page.tsxindex 1e8044f5a1..0cb7c81d3c 100644--- a/frontend/src/components/pages/gallery-page.tsx+++ b/frontend/src/components/pages/gallery-page.tsx@@ -4,6 +4,7 @@ import { SearchIcon } from "lucide-react";import type React from "react";import { Suspense, useMemo, useState } from "react";import { ErrorBoundary } from "@/components/editor/boundary/ErrorBoundary";+import { TruncatedWorkspaceBanner } from "@/components/home/components";import { Spinner } from "@/components/icons/spinner";import { Card, CardContent } from "@/components/ui/card";import { Input } from "@/components/ui/input";@@ -119,10 +120,7 @@ const GalleryPage: React.FC = () => {<ErrorBoundary><div className="flex flex-col gap-2">{workspace.hasMore && (- <Banner kind="warn" className="rounded p-4">- Showing first {workspace.fileCount} files. Your workspace has- more files.- </Banner>+ <TruncatedWorkspaceBanner fileCount={workspace.fileCount} />)}{formattedFiles.length > SEARCH_THRESHOLD && (<Inputdiff --git a/frontend/src/components/pages/home-page.tsx b/frontend/src/components/pages/home-page.tsxindex 763db0813e..aba169f0a3 100644--- a/frontend/src/components/pages/home-page.tsx+++ b/frontend/src/components/pages/home-page.tsx@@ -67,6 +67,7 @@ import {Header,OpenTutorialDropDown,ResourceLinks,+ TruncatedWorkspaceBanner,} from "../home/components";import {expandedFoldersAtom,@@ -196,10 +197,7 @@ const WorkspaceNotebooks: React.FC<{ onRefreshRecents: () => void }> = ({<WorkspaceContext value={workspaceContextValue}><div className="flex flex-col gap-2">{workspace.hasMore && (- <Banner kind="warn" className="rounded p-4">- Showing first {workspace.fileCount} files. Your workspace has more- files.- </Banner>+ <TruncatedWorkspaceBanner fileCount={workspace.fileCount} />)}<HeaderIcon={BookTextIcon}diff --git a/marimo/_server/api/endpoints/home.py b/marimo/_server/api/endpoints/home.pyindex 1acee1bf93..d055634202 100644--- a/marimo/_server/api/endpoints/home.py+++ b/marimo/_server/api/endpoints/home.py@@ -153,7 +153,9 @@ async def workspace_files(marimo_files = await asyncio.to_thread(get_files_with_metadata)file_count = len(marimo_files)- has_more = file_count >= MAX_FILES+ has_more = (+ file_count >= MAX_FILES or session_manager.workspace.is_truncated+ )return WorkspaceFilesResponse(files=marimo_files,root=session_manager.workspace.directory or "",@@ -171,7 +173,9 @@ async def workspace_files(files = await asyncio.to_thread(lambda: session_manager.workspace.files)file_count = count_files(files)- has_more = file_count >= MAX_FILES+ has_more = (+ file_count >= MAX_FILES or session_manager.workspace.is_truncated+ )return WorkspaceFilesResponse(files=files,diff --git a/marimo/_server/files/directory_scanner.py b/marimo/_server/files/directory_scanner.pyindex d8c60d45c1..4c8a03fe67 100644--- a/marimo/_server/files/directory_scanner.py+++ b/marimo/_server/files/directory_scanner.py@@ -76,6 +76,7 @@ class DirectoryScanner:- Skip common directories (venv, node_modules, etc.)- File count limits and timeouts- Marimo app detection+ - Reports (via `truncated`) when a limit cut the walk short"""MAX_DEPTH = 5@@ -138,6 +139,11 @@ class DirectoryScanner:)# Stores partial results in case of timeoutself.partial_results: list[FileInfo] = []+ # Set when a limit (depth, file count, time) or an unreadable entry+ # stopped the walk before the tree was fully explored. Lets callers+ # tell "this directory has no notebooks" apart from "we stopped+ # looking".+ self.truncated = False@propertydef allowed_extensions(self) -> tuple[str, ...]:@@ -159,9 +165,11 @@ class DirectoryScanner:start_time = time.time()file_count = [0] # Use list for closure mutabilityself.partial_results = [] # Reset partial results+ self.truncated = False # Reset truncation flagdef recurse(directory: str, depth: int = 0) -> list[FileInfo] | None:if depth > self.max_depth:+ self.truncated = Truereturn None# Check file limit@@ -169,10 +177,12 @@ class DirectoryScanner:LOGGER.warning(f"Reached maximum file limit ({self.max_files})")+ self.truncated = Truereturn Noneif time.time() - start_time > self.max_execution_time:# Store accumulated results before raising timeout+ self.truncated = Trueraise HTTPException(status_code=HTTPStatus.REQUEST_TIMEOUT,detail=f"Request timed out: Loading workspace files took too long. Showing first {file_count[0]} files.",@@ -182,6 +192,7 @@ class DirectoryScanner:entries = os.scandir(directory)except OSError as e:LOGGER.debug("OSError scanning directory: %s", str(e))+ self.truncated = Truereturn Nonefiles: list[FileInfo] = []@@ -191,6 +202,7 @@ class DirectoryScanner:# Check the limit here, not just after adding a file: a# sibling directory's recursion may have reached it.if file_count[0] >= self.max_files:+ self.truncated = Truebreak# Skip hidden files and directories@@ -206,9 +218,10 @@ class DirectoryScanner:if (entry.name in self.SKIP_DIRSor entry.name.lower() in self.SKIP_DIRS- or depth == self.max_depth):continue+ # Past `max_depth` this returns `None` without any+ # I/O, and records that the walk was cut short.children = recurse(entry.path, depth + 1)if children:entry_path = Path(entry.path)@@ -247,6 +260,9 @@ class DirectoryScanner:LOGGER.debug("Error processing entry %s: %s", entry.path, e)+ # We don't know what this entry was, so it may have been a+ # notebook or a folder holding notebooks.+ self.truncated = Truecontinue# Sort folders then files, based on natural sort (alpha, then num)diff --git a/marimo/_server/models/home.py b/marimo/_server/models/home.pyindex f0f1beca40..3aede5b2a7 100644--- a/marimo/_server/models/home.py+++ b/marimo/_server/models/home.py@@ -41,7 +41,8 @@ class WorkspaceFilesRequest(msgspec.Struct, rename="camel"):class WorkspaceFilesResponse(msgspec.Struct, rename="camel"):root: strfiles: list[FileInfo]- # Indicates if limit was reached+ # Indicates the listing is incomplete: a file count, depth, or time limit+ # was reached while scanning, so some files or folders were left outhas_more: bool = False# Total files foundfile_count: int = 0diff --git a/marimo/_server/workspace/_base.py b/marimo/_server/workspace/_base.pyindex 148ae753c3..80adccb8a0 100644--- a/marimo/_server/workspace/_base.py+++ b/marimo/_server/workspace/_base.py@@ -45,6 +45,16 @@ class NotebookWorkspace(abc.ABC):def files(self) -> list[FileInfo]:"""All files in this workspace as a recursive tree."""+ @property+ def is_truncated(self) -> bool:+ """Whether the last listing was cut short and may be incomplete.++ `False` by default — workspaces with a fixed set of notebooks always+ list all of them. `DirectoryWorkspace` overrides, since its scan is+ bounded by depth, file count, and time limits.+ """+ return False+@abc.abstractmethoddef single_file(self) -> MarimoFile | None:"""If this workspace represents a single notebook, return it."""diff --git a/marimo/_server/workspace/_directory.py b/marimo/_server/workspace/_directory.pyindex 499ecdbe3d..568881435b 100644--- a/marimo/_server/workspace/_directory.py+++ b/marimo/_server/workspace/_directory.py@@ -107,3 +107,14 @@ class DirectoryWorkspace(NotebookWorkspace):else:raisereturn self._lazy_files++ @property+ def is_truncated(self) -> bool:+ """Whether the scan hit a depth, file count, or time limit.++ Notebooks nested deeper than the scanner's depth limit are dropped+ along with their folders, which otherwise looks identical to an empty+ directory; this lets the client say so instead of silently omitting+ them.+ """+ return self._scanner.truncateddiff --git a/tests/_server/test_directory_scanner.py b/tests/_server/test_directory_scanner.pyindex c0a9d4b28c..9e5619534c 100644--- a/tests/_server/test_directory_scanner.py+++ b/tests/_server/test_directory_scanner.py@@ -206,6 +206,81 @@ class TestDirectoryScanner:files = DirectoryScanner(str(test_dir)).scan()assert "long_docstring_app.py" in _file_names(files)+ def test_truncated_when_folders_are_deeper_than_max_depth(+ self, tmp_path: Path+ ):+ """Notebooks past `max_depth` are dropped, but not silently (#10064).++ The folder holding them looks exactly like an empty folder, so the+ scanner has to report that it stopped looking.+ """+ deep = tmp_path / "projects" / "analysis"+ deep.mkdir(parents=True)+ _write(deep / "app.py", MARIMO_APP)++ scanner = DirectoryScanner(str(tmp_path), max_depth=1)+ files = scanner.scan()++ assert _count_files(files) == 0+ assert scanner.truncated++ def test_not_truncated_when_tree_is_fully_scanned(self, test_dir: Path):+ scanner = DirectoryScanner(str(test_dir))+ scanner.scan()+ assert not scanner.truncated++ def test_truncated_when_max_files_reached(self, test_dir: Path):+ for i in range(10):+ _write(test_dir / f"app{i + 3}.py", MARIMO_APP)+ scanner = DirectoryScanner(str(test_dir), max_files=5)+ scanner.scan()+ assert scanner.truncated++ def test_truncated_is_reset_between_scans(self, tmp_path: Path):+ deep = tmp_path / "projects" / "analysis"+ deep.mkdir(parents=True)+ _write(deep / "app.py", MARIMO_APP)++ scanner = DirectoryScanner(str(tmp_path), max_depth=1)+ scanner.scan()+ assert scanner.truncated++ # Deep enough to reach the notebook: no truncation this time.+ scanner.max_depth = 2+ files = scanner.scan()+ assert _count_files(files) == 1+ assert not scanner.truncated++ def test_truncated_when_an_entry_cannot_be_read(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+ ):+ """An entry we fail to stat may have been a notebook, or a folder+ holding notebooks, so it counts as a truncated scan too."""+ import marimo._server.files.directory_scanner as scanner_mod++ (tmp_path / "projects").mkdir()+ _write(tmp_path / "projects" / "app.py", MARIMO_APP)++ class UnreadableEntry:+ name = "projects"+ path = str(tmp_path / "projects")++ def is_symlink(self) -> bool:+ return False++ def is_dir(self) -> bool:+ raise PermissionError("denied")++ monkeypatch.setattr(+ scanner_mod.os, "scandir", lambda _path: iter([UnreadableEntry()])+ )++ scanner = DirectoryScanner(str(tmp_path))+ files = scanner.scan()++ assert files == []+ assert scanner.truncated+def test_partial_results_populated_during_scan(self, test_dir: Path):scanner = DirectoryScanner(str(test_dir))assert scanner.partial_results == []diff --git a/tests/_server/test_workspace.py b/tests/_server/test_workspace.pyindex 73d9401e0b..786354876b 100644--- a/tests/_server/test_workspace.py+++ b/tests/_server/test_workspace.py@@ -750,6 +750,28 @@ def test_lazy_router_respects_max_files(tmp_path: Path):DirectoryScanner.MAX_FILES = original_max_files+def test_lazy_router_reports_truncated_scan(tmp_path: Path):+ """Deeply nested notebooks are omitted, and the workspace says so (#10064)."""+ deep = tmp_path.joinpath("a", "b", "c", "d", "e", "f")+ deep.mkdir(parents=True)+ (deep / "app.py").write_text(file_contents)++ router = DirectoryWorkspace(str(tmp_path), include_markdown=False)++ # The notebook sits past DirectoryScanner.MAX_DEPTH, so nothing is listed.+ assert router.files == []+ assert router.is_truncated+++def test_lazy_router_not_truncated_for_shallow_tree(tmp_path: Path):+ (tmp_path / "app.py").write_text(file_contents)++ router = DirectoryWorkspace(str(tmp_path), include_markdown=False)++ assert len(router.files) == 1+ assert not router.is_truncated++def test_lazy_router_skips_common_dirs(tmp_path: Path):"""Test that DirectoryWorkspace skips common directories"""# Create directories that should be skipped