A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat: Streaming files for check (#6494)
Dylan Madisetti0520399c4f82
diff --git a/marimo/_lint/__init__.py b/marimo/_lint/__init__.pyindex 1756082075..1df953d711 100644--- a/marimo/_lint/__init__.py+++ b/marimo/_lint/__init__.py@@ -7,7 +7,7 @@ from marimo._lint.diagnostic import Diagnostic, Severityfrom marimo._lint.linter import FileStatus, Linterfrom marimo._lint.rule_engine import EarlyStoppingConfig, RuleEnginefrom marimo._lint.rules.base import LintRule-from marimo._utils.files import expand_file_patterns+from marimo._utils.files import async_expand_file_patternsif TYPE_CHECKING:from collections.abc import Callable@@ -43,8 +43,8 @@ def run_check(Returns:Linter with per-file status and diagnostics"""- # Expand patterns to actual files- files_to_check = expand_file_patterns(file_patterns)+ # Get async generator for files+ files_to_check = async_expand_file_patterns(file_patterns)linter = Linter(pipe=pipe,@@ -100,7 +100,7 @@ def collect_messages(rules=filtered_rules,)- files_to_check = expand_file_patterns(file_patterns)+ files_to_check = async_expand_file_patterns(file_patterns)linter.run_streaming(files_to_check)return linter, "\n".join(messages)diff --git a/marimo/_lint/linter.py b/marimo/_lint/linter.pyindex 3ce31e4342..c6688344a5 100644--- a/marimo/_lint/linter.py+++ b/marimo/_lint/linter.py@@ -3,9 +3,10 @@ from __future__ import annotationsimport asyncioimport re+from collections.abc import AsyncIterator, Iteratorfrom dataclasses import dataclass, fieldfrom pathlib import Path-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Unionfrom marimo._ast.load import get_notebook_statusfrom marimo._ast.parse import MarimoFileError@@ -17,7 +18,7 @@ from marimo._loggers import capture_outputfrom marimo._schemas.serialization import NotebookSerializationif TYPE_CHECKING:- from collections.abc import AsyncIterator, Callable+ from collections.abc import AsyncIterator, Callable, Iteratorfrom marimo._lint.rules.base import LintRule@@ -40,6 +41,20 @@ def _contents_differ_excluding_generated_with(return orig_cleaned != gen_cleaned+async def _to_async_iterator(+ files_to_check: Union[AsyncIterator[Path], Iterator[Path]],+) -> AsyncIterator[Path]:+ """Convert a regular iterator to an async iterator if needed."""+ if hasattr(files_to_check, "__aiter__"):+ # Already an async iterator+ async for file_path in files_to_check:+ yield file_path+ else:+ # Convert regular iterator to async+ for file_path in files_to_check:+ yield file_path++@dataclassclass FileStatus:"""Processing status and results for a single file."""@@ -91,105 +106,110 @@ class Linter:self.fixed_count: int = 0self.issues_count: int = 0- async def _run_stream(- self, files_to_check: list[Path]- ) -> AsyncIterator[FileStatus]:- """Asynchronously check files and yield results as they complete."""+ async def _process_single_file(self, file: Path) -> FileStatus:+ """Process a single file and return its status."""+ file_path = str(file)+ file_status = FileStatus(file=file_path)+ # Check if file exists first+ if not file.exists():+ self.errored = True+ file_status.failed = True+ file_status.message = f"File not found: {file_path}"+ file_status.details = [+ f"FileNotFoundError: No such file or directory: '{file_path}'"+ ]+ return file_status- # Create tasks for all files- async def process_file(file: Path) -> FileStatus:- file_path = str(file)- file_status = FileStatus(file=file_path)+ # Check if file is a supported notebook format+ if not file_path.endswith((".py", ".md", ".qmd")):+ file_status.skipped = True+ file_status.message = f"Skipped: {file_path} (not a notebook file)"+ return file_status- # Check if file is a supported notebook format- if not file_path.endswith((".py", ".md", ".qmd")):+ try:+ with capture_output() as (stdout, stderr, logs):+ load_result = get_notebook_status(file_path)+ except SyntaxError as e:+ # Handle syntax errors in notebooks+ self.errored = True+ file_status.failed = True+ file_status.message = f"Failed to parse: {file_path}"+ file_status.details = [f"SyntaxError: {str(e)}"]+ return file_status+ except MarimoFileError as e:+ # Handle syntax errors in notebooks+ if self.ignore_scripts:+ # Skip this file silently when ignore_scripts is enabledfile_status.skipped = Truefile_status.message = (- f"Skipped: {file_path} (not a notebook file)"+ f"Skipped: {file_path} (not a marimo notebook)")return file_status-- try:- with capture_output() as (stdout, stderr, logs):- load_result = get_notebook_status(file_path)- except SyntaxError as e:- # Handle syntax errors in notebooks+ else:self.errored = Truefile_status.failed = True- file_status.message = f"Failed to parse: {file_path}"- file_status.details = [f"SyntaxError: {str(e)}"]+ file_status.message = (+ f"Not recognizable as a marimo notebook: {file_path}"+ )+ file_status.details = [f"MarimoFileError: {str(e)}"]return file_status- except MarimoFileError as e:- # Handle syntax errors in notebooks- if self.ignore_scripts:- # Skip this file silently when ignore_scripts is enabled- file_status.skipped = True- file_status.message = (- f"Skipped: {file_path} (not a marimo notebook)"- )- return file_status- else:- self.errored = True- file_status.failed = True- file_status.message = (- f"Not recognizable as a marimo notebook: {file_path}"- )- file_status.details = [f"MarimoFileError: {str(e)}"]- return file_status- file_status.notebook = load_result.notebook- file_status.contents = load_result.contents+ file_status.notebook = load_result.notebook+ file_status.contents = load_result.contents- if load_result.status == "empty":+ if load_result.status == "empty":+ file_status.skipped = True+ file_status.message = f"Skipped: {file_path} (empty file)"+ elif load_result.status == "invalid":+ if self.ignore_scripts:+ # Skip this file silently when ignore_scripts is enabledfile_status.skipped = True- file_status.message = f"Skipped: {file_path} (empty file)"- elif load_result.status == "invalid":- if self.ignore_scripts:- # Skip this file silently when ignore_scripts is enabled- file_status.skipped = True- file_status.message = (- f"Skipped: {file_path} (not a marimo notebook)"- )- return file_status- else:- file_status.failed = True- file_status.message = (- f"Failed to parse: {file_path} (not a valid notebook)"- )- elif load_result.notebook is not None:- try:- # Check notebook with all rules including parsing- file_status.diagnostics = (- await self.rule_engine.check_notebook(- load_result.notebook,- # Add parsing rule if there's captured output- stdout=stdout.getvalue().strip(),- stderr=stderr.getvalue().strip(),- logs=logs,- )- )- except Exception as e:- # Handle other parsing errors- self.errored = True- file_status.failed = True- file_status.message = f"Failed to process {file_path}"- file_status.details = [str(e)]+ file_status.message = (+ f"Skipped: {file_path} (not a marimo notebook)"+ )+ return file_statuselse:- # Status is valid but no notebook - shouldn't happen but handle gracefully- file_status.skipped = True+ file_status.failed = Truefile_status.message = (- f"Skipped: {file_path} (no notebook content)"+ f"Failed to parse: {file_path} (not a valid notebook)")+ elif load_result.notebook is not None:+ try:+ # Check notebook with all rules including parsing+ file_status.diagnostics = (+ await self.rule_engine.check_notebook(+ load_result.notebook,+ # Add parsing rule if there's captured output+ stdout=stdout.getvalue().strip(),+ stderr=stderr.getvalue().strip(),+ logs=logs,+ )+ )+ except Exception as e:+ # Handle other parsing errors+ self.errored = True+ file_status.failed = True+ file_status.message = f"Failed to process {file_path}"+ file_status.details = [str(e)]+ else:+ # Status is valid but no notebook - shouldn't happen but handle gracefully+ file_status.skipped = True+ file_status.message = f"Skipped: {file_path} (no notebook content)"- # Ensure diagnostics list is initialized for cases where no processing happened- if not hasattr(file_status, "diagnostics"):- file_status.diagnostics = []+ # Ensure diagnostics list is initialized for cases where no processing happened+ if not hasattr(file_status, "diagnostics"):+ file_status.diagnostics = []- return file_status+ return file_status++ async def _run_stream(+ self, files_to_check: list[Path]+ ) -> AsyncIterator[FileStatus]:+ """Asynchronously check files and yield results as they complete."""# Create tasks for all filestasks = [- asyncio.create_task(process_file(file_path))+ asyncio.create_task(self._process_single_file(file_path))for file_path in files_to_check]@@ -254,15 +274,22 @@ class Linter:file_status.notebook, file_status.file)- def run_streaming(self, files_to_check: list[Path]) -> None:+ def run_streaming(+ self, files_to_check: Union[AsyncIterator[Path], Iterator[Path]]+ ) -> None:"""Run linting checks with real-time streaming output."""asyncio.run(self._run_streaming_async(files_to_check))- async def _run_streaming_async(self, files_to_check: list[Path]) -> None:+ async def _run_streaming_async(+ self, files_to_check: Union[AsyncIterator[Path], Iterator[Path]]+ ) -> None:"""Internal async implementation of run_streaming."""# Process files as they completefixed_count = 0- async for file_status in self._run_stream(files_to_check):++ # Convert to async iterator and process+ async for file_path in _to_async_iterator(files_to_check):+ file_status = await self._process_single_file(file_path)self.files.append(file_status)# Stream output via pipe if available@@ -278,6 +305,7 @@ class Linter:fixed_count += 1if self.pipe:self.pipe(f"Updated: {file_status.file}")+self.fixed_count = fixed_countasync def fix(self, file_status: FileStatus) -> bool:diff --git a/marimo/_utils/files.py b/marimo/_utils/files.pyindex 2ad5a1394f..55e205c0ba 100644--- a/marimo/_utils/files.py+++ b/marimo/_utils/files.py@@ -2,7 +2,7 @@import fnmatchimport osimport re-from collections.abc import Generator+from collections.abc import AsyncGenerator, Generatorfrom pathlib import Pathfrom typing import Union@@ -27,6 +27,17 @@ def get_files(folder: str) -> Generator[Path, None, None]:yield from get_files(item.path)+async def async_get_files(folder: str) -> AsyncGenerator[Path, None]:+ """Asynchronously recursively get all files from a folder."""+ with os.scandir(folder) as scan:+ for item in scan:+ if item.is_file():+ yield Path(item.path)+ elif item.is_dir() and not item.name.startswith("."):+ async for file_path in async_get_files(item.path):+ yield file_path++def expand_file_patterns(file_patterns: tuple[str, ...]) -> list[Path]:"""Expand file patterns to actual file paths.@@ -70,3 +81,57 @@ def expand_file_patterns(file_patterns: tuple[str, ...]) -> list[Path]:# Remove duplicates and sortreturn sorted(set(files_to_check))+++async def async_expand_file_patterns(+ file_patterns: tuple[str, ...],+) -> AsyncGenerator[Path, None]:+ """Asynchronously expand file patterns to file paths, yielding as discovered.++ Args:+ file_patterns: Tuple of file patterns (files, directories, or glob-like patterns)++ Yields:+ Path objects for matching files (including non-existent explicit files)+ """+ seen = set()++ for pattern in file_patterns:+ if os.path.isfile(pattern):+ path = Path(pattern)+ if path not in seen:+ seen.add(path)+ yield path+ elif os.path.isdir(pattern):+ async for file_path in async_get_files(pattern):+ if file_path not in seen:+ seen.add(file_path)+ yield file_path+ else:+ # Handle glob patterns by walking from root and filtering+ if "**" in pattern or "*" in pattern or "?" in pattern:+ # Extract root directory to walk from+ root = "."+ parts = pattern.split("/")+ for i, part in enumerate(parts):+ if "*" in part or "?" in part:+ root = "/".join(parts[:i]) if i > 0 else "."+ break+ elif os.path.isdir("/".join(parts[: i + 1])):+ root = "/".join(parts[: i + 1])++ # Get all files from root and filter by pattern+ if os.path.isdir(root):+ async for file_path in async_get_files(root):+ if (+ fnmatch.fnmatch(str(file_path), pattern)+ and file_path not in seen+ ):+ seen.add(file_path)+ yield file_path+ else:+ # Not a glob pattern but file doesn't exist - yield it anyway for error handling+ path = Path(pattern)+ if path not in seen:+ seen.add(path)+ yield pathdiff --git a/tests/_lint/test_run_check.py b/tests/_lint/test_run_check.pyindex 81d36552cd..4c1049d507 100644--- a/tests/_lint/test_run_check.py+++ b/tests/_lint/test_run_check.py@@ -368,4 +368,20 @@ def __():assert result.files[0].failed is True# Note: errored might not be True as we changed error handlingassert "Failed to parse" in result.files[0].message- """Test error handling in run_check."""++ def test_run_check_with_nonexistent_file_pattern(self):+ """Test run_check with a specific nonexistent file."""+ result = run_check(("nonexistent_file.py",))++ assert isinstance(result, Linter)+ assert len(result.files) == 1 # Should create a failed file status+ assert result.files[0].failed is True+ assert "File not found" in result.files[0].message++ def test_run_check_with_nonexistent_directory_pattern(self):+ """Test run_check with nonexistent directory patterns."""+ result = run_check(("nonexistent_dir/**/*.py",))++ assert isinstance(result, Linter)+ assert len(result.files) == 0+ assert result.errored is False