Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

refactor: group kernel streams into KernelStreams; phase-key NotebookCellHooks (#9571)

Myles Scolnick73ec9d2180c7
diff --git a/marimo/_messaging/types.py b/marimo/_messaging/types.py
index 0b1675bcf2..f3629e011c 100644
--- a/marimo/_messaging/types.py
+++ b/marimo/_messaging/types.py
@@ -2,6 +2,7 @@
from __future__ import annotations
import abc
+import dataclasses
import io
from typing import NewType
@@ -103,3 +104,13 @@ class Stdin(io.TextIOBase):
def _stop(self) -> None:
"""Tear down resources, if any."""
+
+
[email protected](kw_only=True)
+class KernelStreams:
+ """The four I/O channels the kernel uses to communicate with the host."""
+
+ stream: Stream
+ stdout: Stdout | None
+ stderr: Stderr | None
+ stdin: Stdin | None
diff --git a/marimo/_pyodide/pyodide_session.py b/marimo/_pyodide/pyodide_session.py
index a6c0336ff9..c056e217d9 100644
--- a/marimo/_pyodide/pyodide_session.py
+++ b/marimo/_pyodide/pyodide_session.py
@@ -17,6 +17,7 @@ from marimo._config.config import (
)
from marimo._convert.markdown import convert_from_ir_to_markdown
from marimo._messaging.msgspec_encoder import encode_json_str
+from marimo._messaging.types import KernelStreams
from marimo._pyodide.restartable_task import RestartableTask
from marimo._pyodide.streams import (
PyodideStderr,
@@ -464,10 +465,9 @@ def _launch_pyodide_kernel(
kernel, ctx = create_kernel(
KernelArgs(
- stream=stream,
- stdout=stdout,
- stderr=stderr,
- stdin=stdin,
+ streams=KernelStreams(
+ stream=stream, stdout=stdout, stderr=stderr, stdin=stdin
+ ),
debugger=debugger,
configs=configs,
app_metadata=app_metadata,
diff --git a/marimo/_runtime/app/kernel_runner.py b/marimo/_runtime/app/kernel_runner.py
index 00e7c81104..eb567035db 100644
--- a/marimo/_runtime/app/kernel_runner.py
+++ b/marimo/_runtime/app/kernel_runner.py
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any
from marimo._ast.cell import CellImpl
from marimo._config.config import DEFAULT_CONFIG
from marimo._dependencies.dependencies import DependencyManager
+from marimo._messaging.types import KernelStreams
from marimo._runtime.app.common import RunOutput
from marimo._runtime.commands import (
AppMetadata,
@@ -108,6 +109,13 @@ class AppKernelRunner:
hooks.add_post_execution(cache_output)
hooks.add_post_execution(_reset_matplotlib_context)
+ streams = KernelStreams(
+ stream=ctx.stream,
+ stdout=None,
+ stderr=None,
+ stdin=None,
+ )
+
self._kernel = Kernel(
cell_configs={},
app_metadata=AppMetadata(
@@ -117,10 +125,7 @@ class AppKernelRunner:
filename=filename,
app_config=app.config,
),
- stream=ctx.stream,
- stdout=None,
- stderr=None,
- stdin=None,
+ streams=streams,
module=create_main_module(
filename,
input_override=None,
@@ -137,9 +142,7 @@ class AppKernelRunner:
self._runtime_context = create_kernel_context(
kernel=self._kernel,
app=app,
- stream=ctx.stream,
- stdout=None,
- stderr=None,
+ streams=streams,
virtual_file_storage="shared_memory",
mode=SessionMode.EDIT,
parent=ctx,
diff --git a/marimo/_runtime/context/kernel_context.py b/marimo/_runtime/context/kernel_context.py
index a4e7ba3dd5..9f99a02402 100644
--- a/marimo/_runtime/context/kernel_context.py
+++ b/marimo/_runtime/context/kernel_context.py
@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any
from marimo._ast.app import AppKernelRunnerRegistry
from marimo._config.config import MarimoConfig
-from marimo._messaging.types import Stderr, Stdout
from marimo._plugins.ui._core.ids import IDProvider, NoIDProviderException
from marimo._runtime.cell_lifecycle_registry import CellLifecycleRegistry
from marimo._runtime.context.types import (
@@ -24,7 +23,7 @@ if TYPE_CHECKING:
from collections.abc import Iterator
from marimo._ast.app import InternalApp
- from marimo._messaging.types import Stream
+ from marimo._messaging.types import KernelStreams
from marimo._runtime.runtime import Kernel
from marimo._runtime.state import State
from marimo._runtime.virtual_file import VirtualFileStorageType
@@ -141,9 +140,7 @@ class KernelRuntimeContext(RuntimeContext):
def create_kernel_context(
*,
kernel: Kernel,
- stream: Stream,
- stdout: Stdout | None,
- stderr: Stderr | None,
+ streams: KernelStreams,
virtual_file_storage: VirtualFileStorageType | None,
mode: SessionMode,
app: InternalApp | None = None,
@@ -185,9 +182,9 @@ def create_kernel_context(
),
virtual_file_registry=VirtualFileRegistry(storage=storage),
virtual_files_supported=virtual_file_storage is not None,
- stream=stream,
- stdout=stdout,
- stderr=stderr,
+ stream=streams.stream,
+ stdout=streams.stdout,
+ stderr=streams.stderr,
children=[],
parent=parent,
filename=kernel.app_metadata.filename,
@@ -198,9 +195,7 @@ def create_kernel_context(
def initialize_kernel_context(
*,
kernel: Kernel,
- stream: Stream,
- stdout: Stdout | None,
- stderr: Stderr | None,
+ streams: KernelStreams,
virtual_file_storage: VirtualFileStorageType | None,
mode: SessionMode,
) -> KernelRuntimeContext:
@@ -210,9 +205,7 @@ def initialize_kernel_context(
"""
ctx = create_kernel_context(
kernel=kernel,
- stream=stream,
- stdout=stdout,
- stderr=stderr,
+ streams=streams,
virtual_file_storage=virtual_file_storage,
mode=mode,
)
diff --git a/marimo/_runtime/kernel_lifecycle.py b/marimo/_runtime/kernel_lifecycle.py
index 7b2264dd6a..c26a6db12f 100644
--- a/marimo/_runtime/kernel_lifecycle.py
+++ b/marimo/_runtime/kernel_lifecycle.py
@@ -40,7 +40,7 @@ if TYPE_CHECKING:
from marimo._ast.cell import CellConfig
from marimo._config.config import MarimoConfig
- from marimo._messaging.types import Stderr, Stdin, Stdout, Stream
+ from marimo._messaging.types import KernelStreams
from marimo._runtime import marimo_pdb
from marimo._runtime.commands import (
AppMetadata,
@@ -62,10 +62,7 @@ if TYPE_CHECKING:
class KernelArgs:
"""All inputs needed to construct a `Kernel` and its runtime context."""
- stream: Stream
- stdout: Stdout | None
- stderr: Stderr | None
- stdin: Stdin | None
+ streams: KernelStreams
debugger: marimo_pdb.MarimoPdb | None
configs: dict[CellId_t, CellConfig]
app_metadata: AppMetadata
@@ -127,10 +124,7 @@ def create_kernel(
kernel = Kernel(
cell_configs=args.configs,
app_metadata=args.app_metadata,
- stream=args.stream,
- stdout=args.stdout,
- stderr=args.stderr,
- stdin=args.stdin,
+ streams=args.streams,
module=patches.patch_main_module(
file=args.app_metadata.filename,
input_override=input_override,
@@ -144,9 +138,7 @@ def create_kernel(
)
ctx = initialize_kernel_context(
kernel=kernel,
- stream=args.stream,
- stdout=args.stdout,
- stderr=args.stderr,
+ streams=args.streams,
virtual_file_storage=args.virtual_file_storage,
mode=args.mode,
)
diff --git a/marimo/_runtime/runner/hooks.py b/marimo/_runtime/runner/hooks.py
index 9f5283e269..c6441b0d0c 100644
--- a/marimo/_runtime/runner/hooks.py
+++ b/marimo/_runtime/runner/hooks.py
@@ -3,8 +3,8 @@ from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass, field
-from enum import IntEnum
-from typing import TYPE_CHECKING
+from enum import Enum, IntEnum
+from typing import TYPE_CHECKING, cast
if TYPE_CHECKING:
from collections.abc import Sequence
@@ -19,6 +19,7 @@ if TYPE_CHECKING:
)
__all__ = [
+ "HookPhase",
"NotebookCellHooks",
"OnFinishHook",
"PostExecutionHook",
@@ -37,6 +38,15 @@ PostExecutionHook = Callable[
OnFinishHook = Callable[["OnFinishHookContext"], None]
+class HookPhase(str, Enum):
+ """Lifecycle phases a hook may register against."""
+
+ PREPARATION = "preparation"
+ PRE_EXECUTION = "pre_execution"
+ POST_EXECUTION = "post_execution"
+ ON_FINISH = "on_finish"
+
+
class Priority(IntEnum):
"""Hook execution priority (lower values run first)."""
@@ -90,59 +100,68 @@ class NotebookCellHooks:
"""
def __init__(
+ self, _lists: dict[HookPhase, _HookList] | None = None
+ ) -> None:
+ self._lists: dict[HookPhase, _HookList] = _lists or {
+ phase: _HookList() for phase in HookPhase
+ }
+
+ def _add(
self,
- _preparation: _HookList | None = None,
- _pre_execution: _HookList | None = None,
- _post_execution: _HookList | None = None,
- _on_finish: _HookList | None = None,
+ phase: HookPhase,
+ hook: Callable[..., None],
+ priority: Priority,
) -> None:
- self._preparation = _preparation or _HookList()
- self._pre_execution = _pre_execution or _HookList()
- self._post_execution = _post_execution or _HookList()
- self._on_finish = _on_finish or _HookList()
+ self._lists[phase].add(hook, priority)
+
+ def _get(self, phase: HookPhase) -> Sequence[Callable[..., None]]:
+ return self._lists[phase].sorted_hooks
def add_preparation(
self, hook: PreparationHook, priority: Priority = Priority.NORMAL
) -> None:
- self._preparation.add(hook, priority)
+ self._add(HookPhase.PREPARATION, hook, priority)
def add_pre_execution(
self, hook: PreExecutionHook, priority: Priority = Priority.NORMAL
) -> None:
- self._pre_execution.add(hook, priority)
+ self._add(HookPhase.PRE_EXECUTION, hook, priority)
def add_post_execution(
self, hook: PostExecutionHook, priority: Priority = Priority.NORMAL
) -> None:
- self._post_execution.add(hook, priority)
+ self._add(HookPhase.POST_EXECUTION, hook, priority)
def add_on_finish(
self, hook: OnFinishHook, priority: Priority = Priority.NORMAL
) -> None:
- self._on_finish.add(hook, priority)
+ self._add(HookPhase.ON_FINISH, hook, priority)
@property
def preparation_hooks(self) -> Sequence[PreparationHook]:
- return self._preparation.sorted_hooks
+ return cast(
+ "Sequence[PreparationHook]", self._get(HookPhase.PREPARATION)
+ )
@property
def pre_execution_hooks(self) -> Sequence[PreExecutionHook]:
- return self._pre_execution.sorted_hooks
+ return cast(
+ "Sequence[PreExecutionHook]", self._get(HookPhase.PRE_EXECUTION)
+ )
@property
def post_execution_hooks(self) -> Sequence[PostExecutionHook]:
- return self._post_execution.sorted_hooks
+ return cast(
+ "Sequence[PostExecutionHook]", self._get(HookPhase.POST_EXECUTION)
+ )
@property
def on_finish_hooks(self) -> Sequence[OnFinishHook]:
- return self._on_finish.sorted_hooks
+ return cast("Sequence[OnFinishHook]", self._get(HookPhase.ON_FINISH))
def copy(self) -> NotebookCellHooks:
return NotebookCellHooks(
- _preparation=self._preparation.copy(),
- _pre_execution=self._pre_execution.copy(),
- _post_execution=self._post_execution.copy(),
- _on_finish=self._on_finish.copy(),
+ {phase: lst.copy() for phase, lst in self._lists.items()}
)
@@ -165,17 +184,13 @@ def create_default_hooks() -> NotebookCellHooks:
)
hooks = NotebookCellHooks()
-
- for prep_hook in PREPARATION_HOOKS:
- hooks.add_preparation(prep_hook)
-
- for pre_hook in PRE_EXECUTION_HOOKS:
- hooks.add_pre_execution(pre_hook)
-
- for post_hook in POST_EXECUTION_HOOKS:
- hooks.add_post_execution(post_hook)
-
- for finish_hook in ON_FINISH_HOOKS:
- hooks.add_on_finish(finish_hook)
-
+ defaults: list[tuple[HookPhase, Sequence[Callable[..., None]]]] = [
+ (HookPhase.PREPARATION, PREPARATION_HOOKS),
+ (HookPhase.PRE_EXECUTION, PRE_EXECUTION_HOOKS),
+ (HookPhase.POST_EXECUTION, POST_EXECUTION_HOOKS),
+ (HookPhase.ON_FINISH, ON_FINISH_HOOKS),
+ ]
+ for phase, hook_list in defaults:
+ for hook in hook_list:
+ hooks._add(phase, hook, Priority.NORMAL)
return hooks
diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.py
index be56a50c32..bf0075c600 100644
--- a/marimo/_runtime/runtime.py
+++ b/marimo/_runtime/runtime.py
@@ -108,6 +108,7 @@ from marimo._messaging.streams import (
from marimo._messaging.tracebacks import write_traceback
from marimo._messaging.types import (
KernelMessage,
+ KernelStreams,
Stderr,
Stdin,
Stdout,
@@ -497,10 +498,7 @@ class Kernel:
cell_configs (dict[CellId_t, CellConfig]): Initial configuration for each cell.
app_metadata (AppMetadata): Metadata about the notebook.
user_config (MarimoConfig): The initial user configuration.
- stream (Stream): Object used to communicate with the server/outside world.
- stdout (Stdout | None): Replacement for sys.stdout.
- stderr (Stderr | None): Replacement for sys.stderr.
- stdin (Stdin | None): Replacement for sys.stdin.
+ streams (KernelStreams): The four I/O channels (stream, stdout, stderr, stdin).
module (ModuleType): Module in which to execute code.
enqueue_control_request (Callable[[ControlRequest], None]): Callback to enqueue control requests.
debugger_override (marimo_pdb.MarimoPdb | None): A replacement for the built-in Pdb.
@@ -512,10 +510,7 @@ class Kernel:
cell_configs: dict[CellId_t, CellConfig],
app_metadata: AppMetadata,
user_config: MarimoConfig,
- stream: Stream,
- stdout: Stdout | None,
- stderr: Stderr | None,
- stdin: Stdin | None,
+ streams: KernelStreams,
module: ModuleType,
enqueue_control_request: Callable[[CommandMessage], None],
hooks: NotebookCellHooks,
@@ -538,10 +533,7 @@ class Kernel:
# kernel globals.
sys.argv = self.argv
- self.stream = stream
- self.stdout = stdout
- self.stderr = stderr
- self.stdin = stdin
+ self._streams = streams
self.enqueue_control_request = enqueue_control_request
# timestamp at which most recently processed interrupt was seen;
# the kernel rejects run requests that were issued before that
@@ -658,6 +650,22 @@ class Kernel:
if lifespan.has_lifespans():
self._lifespan = lifespan(None)
+ @property
+ def stream(self) -> Stream:
+ return self._streams.stream
+
+ @property
+ def stdout(self) -> Stdout | None:
+ return self._streams.stdout
+
+ @property
+ def stderr(self) -> Stderr | None:
+ return self._streams.stderr
+
+ @property
+ def stdin(self) -> Stdin | None:
+ return self._streams.stdin
+
def teardown(self) -> None:
"""Teardown resources owned by the kernel."""
if self.stdout is not None:
@@ -3553,7 +3561,9 @@ class RequestHandler:
@dataclasses.dataclass
-class _KernelStreams:
+class _LaunchStreams:
+ """Superset of `KernelStreams` carrying the subprocess bootstrap pieces."""
+
stream: ThreadSafeStream
stdout: ThreadSafeStdout | None
stderr: ThreadSafeStderr | None
@@ -3561,6 +3571,15 @@ class _KernelStreams:
debugger: marimo_pdb.MarimoPdb | None
pipe: TypedConnection[KernelMessage] | None
+ @property
+ def kernel_streams(self) -> KernelStreams:
+ return KernelStreams(
+ stream=self.stream,
+ stdout=self.stdout,
+ stderr=self.stderr,
+ stdin=self.stdin,
+ )
+
def close(self, use_fd_redirect: bool) -> None:
if not use_fd_redirect:
from marimo._messaging.thread_local_streams import (
@@ -3631,7 +3650,7 @@ def _create_streams(
is_edit_mode: bool,
should_redirect_stdio: bool,
use_fd_redirect: bool,
-) -> _KernelStreams | None:
+) -> _LaunchStreams | None:
# Returns None when the socket fails to connect; callers should bail out.
pipe: TypedConnection[KernelMessage] | None = None
if socket_addr is not None:
@@ -3696,7 +3715,7 @@ def _create_streams(
if is_edit_mode and not bool(os.getenv("DEBUGPY_RUNNING"))
else None
)
- return _KernelStreams(
+ return _LaunchStreams(
stream=stream,
stdout=stdout,
stderr=stderr,
@@ -3782,10 +3801,7 @@ def launch_kernel(
with kernel_session(
KernelArgs(
- stream=streams.stream,
- stdout=streams.stdout,
- stderr=streams.stderr,
- stdin=streams.stdin,
+ streams=streams.kernel_streams,
debugger=streams.debugger,
configs=configs,
app_metadata=app_metadata,
diff --git a/tests/_runtime/_helpers/session.py b/tests/_runtime/_helpers/session.py
index 1a30eb9851..0e46bafa22 100644
--- a/tests/_runtime/_helpers/session.py
+++ b/tests/_runtime/_helpers/session.py
@@ -11,10 +11,11 @@ import asyncio
import contextlib
import dataclasses
import sys
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, cast
from marimo._config.config import DEFAULT_CONFIG
from marimo._messaging.print_override import print_override
+from marimo._messaging.types import KernelStreams
from marimo._runtime.kernel_lifecycle import KernelArgs, kernel_session
from marimo._runtime.marimo_pdb import MarimoPdb
from marimo._runtime.virtual_file import VirtualFileStorageType
@@ -44,10 +45,23 @@ class TestKernel:
kernel: Kernel
ctx: KernelRuntimeContext
- stream: MockStream
- stdout: MockStdout
- stderr: MockStderr
- stdin: MockStdin
+ streams: KernelStreams
+
+ @property
+ def stream(self) -> MockStream:
+ return cast(MockStream, self.streams.stream)
+
+ @property
+ def stdout(self) -> MockStdout:
+ return cast(MockStdout, self.streams.stdout)
+
+ @property
+ def stderr(self) -> MockStderr:
+ return cast(MockStderr, self.streams.stderr)
+
+ @property
+ def stdin(self) -> MockStdin:
+ return cast(MockStdin, self.streams.stdin)
@contextlib.contextmanager
@@ -73,6 +87,9 @@ def mocked_kernel_session(
stderr = MockStderr(stream)
stdin = MockStdin(stream)
debugger = MarimoPdb(stdout=stdout, stdin=stdin) if with_debugger else None
+ streams = KernelStreams(
+ stream=stream, stdout=stdout, stderr=stderr, stdin=stdin
+ )
saved_main = sys.modules.get("__main__")
# Kernel.teardown() clears the kernel module's __dict__, which breaks
@@ -84,10 +101,7 @@ def mocked_kernel_session(
)
args = KernelArgs(
- stream=stream,
- stdout=stdout,
- stderr=stderr,
- stdin=stdin,
+ streams=streams,
debugger=debugger,
configs=configs or {},
app_metadata=app_metadata or default_app_metadata(),
@@ -105,14 +119,7 @@ def mocked_kernel_session(
kernel.execution_type = execution_type
if reactive_mode is not None:
kernel.reactive_execution_mode = reactive_mode # type: ignore[assignment]
- yield TestKernel(
- kernel=kernel,
- ctx=ctx,
- stream=stream,
- stdout=stdout,
- stderr=stderr,
- stdin=stdin,
- )
+ yield TestKernel(kernel=kernel, ctx=ctx, streams=streams)
finally:
sys.meta_path[:] = saved_meta_path
if saved_main is not None:
diff --git a/tests/conftest.py b/tests/conftest.py
index d92158950b..bf5e2c29ef 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -343,10 +343,12 @@ def mocked_kernel() -> Generator[MockedKernel, None, None]:
# Installs an execution context without stream redirection
@pytest.fixture
def executing_kernel() -> Generator[Kernel, None, None]:
+ from marimo._messaging.types import KernelStreams
+
mocked = MockedKernel.open()
- mocked.k.stdout = None
- mocked.k.stderr = None
- mocked.k.stdin = None
+ mocked.k._streams = KernelStreams(
+ stream=mocked.k.stream, stdout=None, stderr=None, stdin=None
+ )
with mocked.k._install_execution_context(cell_id="0"):
yield mocked.k
mocked.teardown()