A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
refactor: group kernel streams into KernelStreams; phase-key NotebookCellHooks (#9571)
Myles Scolnick73ec9d2180c7
diff --git a/marimo/_messaging/types.py b/marimo/_messaging/types.pyindex 0b1675bcf2..f3629e011c 100644--- a/marimo/_messaging/types.py+++ b/marimo/_messaging/types.py@@ -2,6 +2,7 @@from __future__ import annotationsimport abc+import dataclassesimport iofrom typing import NewType@@ -103,3 +104,13 @@ class Stdin(io.TextIOBase):def _stop(self) -> None:"""Tear down resources, if any."""++[email protected](kw_only=True)+class KernelStreams:+ """The four I/O channels the kernel uses to communicate with the host."""++ stream: Stream+ stdout: Stdout | None+ stderr: Stderr | None+ stdin: Stdin | Nonediff --git a/marimo/_pyodide/pyodide_session.py b/marimo/_pyodide/pyodide_session.pyindex a6c0336ff9..c056e217d9 100644--- a/marimo/_pyodide/pyodide_session.py+++ b/marimo/_pyodide/pyodide_session.py@@ -17,6 +17,7 @@ from marimo._config.config import ()from marimo._convert.markdown import convert_from_ir_to_markdownfrom marimo._messaging.msgspec_encoder import encode_json_str+from marimo._messaging.types import KernelStreamsfrom marimo._pyodide.restartable_task import RestartableTaskfrom marimo._pyodide.streams import (PyodideStderr,@@ -464,10 +465,9 @@ def _launch_pyodide_kernel(kernel, ctx = create_kernel(KernelArgs(- stream=stream,- stdout=stdout,- stderr=stderr,- stdin=stdin,+ streams=KernelStreams(+ stream=stream, stdout=stdout, stderr=stderr, stdin=stdin+ ),debugger=debugger,configs=configs,app_metadata=app_metadata,diff --git a/marimo/_runtime/app/kernel_runner.py b/marimo/_runtime/app/kernel_runner.pyindex 00e7c81104..eb567035db 100644--- a/marimo/_runtime/app/kernel_runner.py+++ b/marimo/_runtime/app/kernel_runner.py@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Anyfrom marimo._ast.cell import CellImplfrom marimo._config.config import DEFAULT_CONFIGfrom marimo._dependencies.dependencies import DependencyManager+from marimo._messaging.types import KernelStreamsfrom marimo._runtime.app.common import RunOutputfrom marimo._runtime.commands import (AppMetadata,@@ -108,6 +109,13 @@ class AppKernelRunner:hooks.add_post_execution(cache_output)hooks.add_post_execution(_reset_matplotlib_context)+ streams = KernelStreams(+ stream=ctx.stream,+ stdout=None,+ stderr=None,+ stdin=None,+ )+self._kernel = Kernel(cell_configs={},app_metadata=AppMetadata(@@ -117,10 +125,7 @@ class AppKernelRunner:filename=filename,app_config=app.config,),- stream=ctx.stream,- stdout=None,- stderr=None,- stdin=None,+ streams=streams,module=create_main_module(filename,input_override=None,@@ -137,9 +142,7 @@ class AppKernelRunner:self._runtime_context = create_kernel_context(kernel=self._kernel,app=app,- stream=ctx.stream,- stdout=None,- stderr=None,+ streams=streams,virtual_file_storage="shared_memory",mode=SessionMode.EDIT,parent=ctx,diff --git a/marimo/_runtime/context/kernel_context.py b/marimo/_runtime/context/kernel_context.pyindex a4e7ba3dd5..9f99a02402 100644--- a/marimo/_runtime/context/kernel_context.py+++ b/marimo/_runtime/context/kernel_context.py@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Anyfrom marimo._ast.app import AppKernelRunnerRegistryfrom marimo._config.config import MarimoConfig-from marimo._messaging.types import Stderr, Stdoutfrom marimo._plugins.ui._core.ids import IDProvider, NoIDProviderExceptionfrom marimo._runtime.cell_lifecycle_registry import CellLifecycleRegistryfrom marimo._runtime.context.types import (@@ -24,7 +23,7 @@ if TYPE_CHECKING:from collections.abc import Iteratorfrom marimo._ast.app import InternalApp- from marimo._messaging.types import Stream+ from marimo._messaging.types import KernelStreamsfrom marimo._runtime.runtime import Kernelfrom marimo._runtime.state import Statefrom marimo._runtime.virtual_file import VirtualFileStorageType@@ -141,9 +140,7 @@ class KernelRuntimeContext(RuntimeContext):def create_kernel_context(*,kernel: Kernel,- stream: Stream,- stdout: Stdout | None,- stderr: Stderr | None,+ streams: KernelStreams,virtual_file_storage: VirtualFileStorageType | None,mode: SessionMode,app: InternalApp | None = None,@@ -185,9 +182,9 @@ def create_kernel_context(),virtual_file_registry=VirtualFileRegistry(storage=storage),virtual_files_supported=virtual_file_storage is not None,- stream=stream,- stdout=stdout,- stderr=stderr,+ stream=streams.stream,+ stdout=streams.stdout,+ stderr=streams.stderr,children=[],parent=parent,filename=kernel.app_metadata.filename,@@ -198,9 +195,7 @@ def create_kernel_context(def initialize_kernel_context(*,kernel: Kernel,- stream: Stream,- stdout: Stdout | None,- stderr: Stderr | None,+ streams: KernelStreams,virtual_file_storage: VirtualFileStorageType | None,mode: SessionMode,) -> KernelRuntimeContext:@@ -210,9 +205,7 @@ def initialize_kernel_context("""ctx = create_kernel_context(kernel=kernel,- stream=stream,- stdout=stdout,- stderr=stderr,+ streams=streams,virtual_file_storage=virtual_file_storage,mode=mode,)diff --git a/marimo/_runtime/kernel_lifecycle.py b/marimo/_runtime/kernel_lifecycle.pyindex 7b2264dd6a..c26a6db12f 100644--- a/marimo/_runtime/kernel_lifecycle.py+++ b/marimo/_runtime/kernel_lifecycle.py@@ -40,7 +40,7 @@ if TYPE_CHECKING:from marimo._ast.cell import CellConfigfrom marimo._config.config import MarimoConfig- from marimo._messaging.types import Stderr, Stdin, Stdout, Stream+ from marimo._messaging.types import KernelStreamsfrom marimo._runtime import marimo_pdbfrom marimo._runtime.commands import (AppMetadata,@@ -62,10 +62,7 @@ if TYPE_CHECKING:class KernelArgs:"""All inputs needed to construct a `Kernel` and its runtime context."""- stream: Stream- stdout: Stdout | None- stderr: Stderr | None- stdin: Stdin | None+ streams: KernelStreamsdebugger: marimo_pdb.MarimoPdb | Noneconfigs: dict[CellId_t, CellConfig]app_metadata: AppMetadata@@ -127,10 +124,7 @@ def create_kernel(kernel = Kernel(cell_configs=args.configs,app_metadata=args.app_metadata,- stream=args.stream,- stdout=args.stdout,- stderr=args.stderr,- stdin=args.stdin,+ streams=args.streams,module=patches.patch_main_module(file=args.app_metadata.filename,input_override=input_override,@@ -144,9 +138,7 @@ def create_kernel()ctx = initialize_kernel_context(kernel=kernel,- stream=args.stream,- stdout=args.stdout,- stderr=args.stderr,+ streams=args.streams,virtual_file_storage=args.virtual_file_storage,mode=args.mode,)diff --git a/marimo/_runtime/runner/hooks.py b/marimo/_runtime/runner/hooks.pyindex 9f5283e269..c6441b0d0c 100644--- a/marimo/_runtime/runner/hooks.py+++ b/marimo/_runtime/runner/hooks.py@@ -3,8 +3,8 @@ from __future__ import annotationsfrom collections.abc import Callablefrom dataclasses import dataclass, field-from enum import IntEnum-from typing import TYPE_CHECKING+from enum import Enum, IntEnum+from typing import TYPE_CHECKING, castif TYPE_CHECKING:from collections.abc import Sequence@@ -19,6 +19,7 @@ if TYPE_CHECKING:)__all__ = [+ "HookPhase","NotebookCellHooks","OnFinishHook","PostExecutionHook",@@ -37,6 +38,15 @@ PostExecutionHook = Callable[OnFinishHook = Callable[["OnFinishHookContext"], None]+class HookPhase(str, Enum):+ """Lifecycle phases a hook may register against."""++ PREPARATION = "preparation"+ PRE_EXECUTION = "pre_execution"+ POST_EXECUTION = "post_execution"+ ON_FINISH = "on_finish"++class Priority(IntEnum):"""Hook execution priority (lower values run first)."""@@ -90,59 +100,68 @@ class NotebookCellHooks:"""def __init__(+ self, _lists: dict[HookPhase, _HookList] | None = None+ ) -> None:+ self._lists: dict[HookPhase, _HookList] = _lists or {+ phase: _HookList() for phase in HookPhase+ }++ def _add(self,- _preparation: _HookList | None = None,- _pre_execution: _HookList | None = None,- _post_execution: _HookList | None = None,- _on_finish: _HookList | None = None,+ phase: HookPhase,+ hook: Callable[..., None],+ priority: Priority,) -> None:- self._preparation = _preparation or _HookList()- self._pre_execution = _pre_execution or _HookList()- self._post_execution = _post_execution or _HookList()- self._on_finish = _on_finish or _HookList()+ self._lists[phase].add(hook, priority)++ def _get(self, phase: HookPhase) -> Sequence[Callable[..., None]]:+ return self._lists[phase].sorted_hooksdef add_preparation(self, hook: PreparationHook, priority: Priority = Priority.NORMAL) -> None:- self._preparation.add(hook, priority)+ self._add(HookPhase.PREPARATION, hook, priority)def add_pre_execution(self, hook: PreExecutionHook, priority: Priority = Priority.NORMAL) -> None:- self._pre_execution.add(hook, priority)+ self._add(HookPhase.PRE_EXECUTION, hook, priority)def add_post_execution(self, hook: PostExecutionHook, priority: Priority = Priority.NORMAL) -> None:- self._post_execution.add(hook, priority)+ self._add(HookPhase.POST_EXECUTION, hook, priority)def add_on_finish(self, hook: OnFinishHook, priority: Priority = Priority.NORMAL) -> None:- self._on_finish.add(hook, priority)+ self._add(HookPhase.ON_FINISH, hook, priority)@propertydef preparation_hooks(self) -> Sequence[PreparationHook]:- return self._preparation.sorted_hooks+ return cast(+ "Sequence[PreparationHook]", self._get(HookPhase.PREPARATION)+ )@propertydef pre_execution_hooks(self) -> Sequence[PreExecutionHook]:- return self._pre_execution.sorted_hooks+ return cast(+ "Sequence[PreExecutionHook]", self._get(HookPhase.PRE_EXECUTION)+ )@propertydef post_execution_hooks(self) -> Sequence[PostExecutionHook]:- return self._post_execution.sorted_hooks+ return cast(+ "Sequence[PostExecutionHook]", self._get(HookPhase.POST_EXECUTION)+ )@propertydef on_finish_hooks(self) -> Sequence[OnFinishHook]:- return self._on_finish.sorted_hooks+ return cast("Sequence[OnFinishHook]", self._get(HookPhase.ON_FINISH))def copy(self) -> NotebookCellHooks:return NotebookCellHooks(- _preparation=self._preparation.copy(),- _pre_execution=self._pre_execution.copy(),- _post_execution=self._post_execution.copy(),- _on_finish=self._on_finish.copy(),+ {phase: lst.copy() for phase, lst in self._lists.items()})@@ -165,17 +184,13 @@ def create_default_hooks() -> NotebookCellHooks:)hooks = NotebookCellHooks()-- for prep_hook in PREPARATION_HOOKS:- hooks.add_preparation(prep_hook)-- for pre_hook in PRE_EXECUTION_HOOKS:- hooks.add_pre_execution(pre_hook)-- for post_hook in POST_EXECUTION_HOOKS:- hooks.add_post_execution(post_hook)-- for finish_hook in ON_FINISH_HOOKS:- hooks.add_on_finish(finish_hook)-+ defaults: list[tuple[HookPhase, Sequence[Callable[..., None]]]] = [+ (HookPhase.PREPARATION, PREPARATION_HOOKS),+ (HookPhase.PRE_EXECUTION, PRE_EXECUTION_HOOKS),+ (HookPhase.POST_EXECUTION, POST_EXECUTION_HOOKS),+ (HookPhase.ON_FINISH, ON_FINISH_HOOKS),+ ]+ for phase, hook_list in defaults:+ for hook in hook_list:+ hooks._add(phase, hook, Priority.NORMAL)return hooksdiff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.pyindex be56a50c32..bf0075c600 100644--- a/marimo/_runtime/runtime.py+++ b/marimo/_runtime/runtime.py@@ -108,6 +108,7 @@ from marimo._messaging.streams import (from marimo._messaging.tracebacks import write_tracebackfrom marimo._messaging.types import (KernelMessage,+ KernelStreams,Stderr,Stdin,Stdout,@@ -497,10 +498,7 @@ class Kernel:cell_configs (dict[CellId_t, CellConfig]): Initial configuration for each cell.app_metadata (AppMetadata): Metadata about the notebook.user_config (MarimoConfig): The initial user configuration.- stream (Stream): Object used to communicate with the server/outside world.- stdout (Stdout | None): Replacement for sys.stdout.- stderr (Stderr | None): Replacement for sys.stderr.- stdin (Stdin | None): Replacement for sys.stdin.+ streams (KernelStreams): The four I/O channels (stream, stdout, stderr, stdin).module (ModuleType): Module in which to execute code.enqueue_control_request (Callable[[ControlRequest], None]): Callback to enqueue control requests.debugger_override (marimo_pdb.MarimoPdb | None): A replacement for the built-in Pdb.@@ -512,10 +510,7 @@ class Kernel:cell_configs: dict[CellId_t, CellConfig],app_metadata: AppMetadata,user_config: MarimoConfig,- stream: Stream,- stdout: Stdout | None,- stderr: Stderr | None,- stdin: Stdin | None,+ streams: KernelStreams,module: ModuleType,enqueue_control_request: Callable[[CommandMessage], None],hooks: NotebookCellHooks,@@ -538,10 +533,7 @@ class Kernel:# kernel globals.sys.argv = self.argv- self.stream = stream- self.stdout = stdout- self.stderr = stderr- self.stdin = stdin+ self._streams = streamsself.enqueue_control_request = enqueue_control_request# timestamp at which most recently processed interrupt was seen;# the kernel rejects run requests that were issued before that@@ -658,6 +650,22 @@ class Kernel:if lifespan.has_lifespans():self._lifespan = lifespan(None)+ @property+ def stream(self) -> Stream:+ return self._streams.stream++ @property+ def stdout(self) -> Stdout | None:+ return self._streams.stdout++ @property+ def stderr(self) -> Stderr | None:+ return self._streams.stderr++ @property+ def stdin(self) -> Stdin | None:+ return self._streams.stdin+def teardown(self) -> None:"""Teardown resources owned by the kernel."""if self.stdout is not None:@@ -3553,7 +3561,9 @@ class RequestHandler:@dataclasses.dataclass-class _KernelStreams:+class _LaunchStreams:+ """Superset of `KernelStreams` carrying the subprocess bootstrap pieces."""+stream: ThreadSafeStreamstdout: ThreadSafeStdout | Nonestderr: ThreadSafeStderr | None@@ -3561,6 +3571,15 @@ class _KernelStreams:debugger: marimo_pdb.MarimoPdb | Nonepipe: TypedConnection[KernelMessage] | None+ @property+ def kernel_streams(self) -> KernelStreams:+ return KernelStreams(+ stream=self.stream,+ stdout=self.stdout,+ stderr=self.stderr,+ stdin=self.stdin,+ )+def close(self, use_fd_redirect: bool) -> None:if not use_fd_redirect:from marimo._messaging.thread_local_streams import (@@ -3631,7 +3650,7 @@ def _create_streams(is_edit_mode: bool,should_redirect_stdio: bool,use_fd_redirect: bool,-) -> _KernelStreams | None:+) -> _LaunchStreams | None:# Returns None when the socket fails to connect; callers should bail out.pipe: TypedConnection[KernelMessage] | None = Noneif socket_addr is not None:@@ -3696,7 +3715,7 @@ def _create_streams(if is_edit_mode and not bool(os.getenv("DEBUGPY_RUNNING"))else None)- return _KernelStreams(+ return _LaunchStreams(stream=stream,stdout=stdout,stderr=stderr,@@ -3782,10 +3801,7 @@ def launch_kernel(with kernel_session(KernelArgs(- stream=streams.stream,- stdout=streams.stdout,- stderr=streams.stderr,- stdin=streams.stdin,+ streams=streams.kernel_streams,debugger=streams.debugger,configs=configs,app_metadata=app_metadata,diff --git a/tests/_runtime/_helpers/session.py b/tests/_runtime/_helpers/session.pyindex 1a30eb9851..0e46bafa22 100644--- a/tests/_runtime/_helpers/session.py+++ b/tests/_runtime/_helpers/session.py@@ -11,10 +11,11 @@ import asyncioimport contextlibimport dataclassesimport sys-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, castfrom marimo._config.config import DEFAULT_CONFIGfrom marimo._messaging.print_override import print_override+from marimo._messaging.types import KernelStreamsfrom marimo._runtime.kernel_lifecycle import KernelArgs, kernel_sessionfrom marimo._runtime.marimo_pdb import MarimoPdbfrom marimo._runtime.virtual_file import VirtualFileStorageType@@ -44,10 +45,23 @@ class TestKernel:kernel: Kernelctx: KernelRuntimeContext- stream: MockStream- stdout: MockStdout- stderr: MockStderr- stdin: MockStdin+ streams: KernelStreams++ @property+ def stream(self) -> MockStream:+ return cast(MockStream, self.streams.stream)++ @property+ def stdout(self) -> MockStdout:+ return cast(MockStdout, self.streams.stdout)++ @property+ def stderr(self) -> MockStderr:+ return cast(MockStderr, self.streams.stderr)++ @property+ def stdin(self) -> MockStdin:+ return cast(MockStdin, self.streams.stdin)@contextlib.contextmanager@@ -73,6 +87,9 @@ def mocked_kernel_session(stderr = MockStderr(stream)stdin = MockStdin(stream)debugger = MarimoPdb(stdout=stdout, stdin=stdin) if with_debugger else None+ streams = KernelStreams(+ stream=stream, stdout=stdout, stderr=stderr, stdin=stdin+ )saved_main = sys.modules.get("__main__")# Kernel.teardown() clears the kernel module's __dict__, which breaks@@ -84,10 +101,7 @@ def mocked_kernel_session()args = KernelArgs(- stream=stream,- stdout=stdout,- stderr=stderr,- stdin=stdin,+ streams=streams,debugger=debugger,configs=configs or {},app_metadata=app_metadata or default_app_metadata(),@@ -105,14 +119,7 @@ def mocked_kernel_session(kernel.execution_type = execution_typeif reactive_mode is not None:kernel.reactive_execution_mode = reactive_mode # type: ignore[assignment]- yield TestKernel(- kernel=kernel,- ctx=ctx,- stream=stream,- stdout=stdout,- stderr=stderr,- stdin=stdin,- )+ yield TestKernel(kernel=kernel, ctx=ctx, streams=streams)finally:sys.meta_path[:] = saved_meta_pathif saved_main is not None:diff --git a/tests/conftest.py b/tests/conftest.pyindex d92158950b..bf5e2c29ef 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -343,10 +343,12 @@ def mocked_kernel() -> Generator[MockedKernel, None, None]:# Installs an execution context without stream redirection@pytest.fixturedef executing_kernel() -> Generator[Kernel, None, None]:+ from marimo._messaging.types import KernelStreams+mocked = MockedKernel.open()- mocked.k.stdout = None- mocked.k.stderr = None- mocked.k.stdin = None+ mocked.k._streams = KernelStreams(+ stream=mocked.k.stream, stdout=None, stderr=None, stdin=None+ )with mocked.k._install_execution_context(cell_id="0"):yield mocked.kmocked.teardown()