A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
refactor: reduce fields exposed on the Session object (#7537)
Myles Scolnicka09592f9f0df
diff --git a/marimo/_server/api/endpoints/health.py b/marimo/_server/api/endpoints/health.pyindex db4c8ae156..576ebd5f4c 100644--- a/marimo/_server/api/endpoints/health.py+++ b/marimo/_server/api/endpoints/health.py@@ -2,7 +2,6 @@from __future__ import annotationsfrom functools import lru_cache-from multiprocessing import Processfrom typing import TYPE_CHECKING, Any, Optionalfrom starlette.authentication import requires@@ -213,10 +212,8 @@ async def usage(request: Request) -> JSONResponse:kernel_memory: Optional[int] = Nonesession = AppState(request).get_current_session()try:- if session and isinstance(session.kernel_manager.kernel_task, Process):- kernel_process = psutil.Process(- session.kernel_manager.kernel_task.pid- )+ if session and (pid := session.kernel_pid()) is not None:+ kernel_process = psutil.Process(pid)kernel_memory = kernel_process.memory_info().rsskernel_children = kernel_process.children(recursive=True)for child in kernel_children:diff --git a/marimo/_server/api/endpoints/ws.py b/marimo/_server/api/endpoints/ws.pyindex 7bdcdd3604..9cf4d9f03c 100644--- a/marimo/_server/api/endpoints/ws.py+++ b/marimo/_server/api/endpoints/ws.py@@ -273,8 +273,11 @@ class WebsocketHandler(SessionConsumer):# to be sent to the frontendself.message_queue: asyncio.Queue[KernelMessage]self.ws_future: asyncio.Future[tuple[None, None]] | None = None+ self._consumer_id = ConsumerId(session_id)- super().__init__(consumer_id=ConsumerId(session_id))+ @property+ def consumer_id(self) -> ConsumerId:+ return self._consumer_iddef _write_kernel_ready(self,diff --git a/marimo/_server/export/__init__.py b/marimo/_server/export/__init__.pyindex 8c1fa5a33a..23d66ab4fb 100644--- a/marimo/_server/export/__init__.py+++ b/marimo/_server/export/__init__.py@@ -244,7 +244,10 @@ async def run_app_until_completion(class DefaultSessionConsumer(SessionConsumer):def __init__(self) -> None:self.did_error = False- super().__init__(consumer_id=ConsumerId("default"))++ @property+ def consumer_id(self) -> ConsumerId:+ return ConsumerId("default")def on_start(self,@@ -349,7 +352,7 @@ async def run_app_until_completion(# might still exist; the better thing to do would be to flush# the worker, then ask it to quit and join on it. If we have an# issue with some outputs being missed, that's what we should do.- session.message_distributor.flush()+ session.flush_messages()# Hack: yield to give the session view a chance to process the incoming# console operations.await asyncio.sleep(0.1)diff --git a/marimo/_server/model.py b/marimo/_server/model.pyindex 9184151f2c..0fea99e85f 100644--- a/marimo/_server/model.py+++ b/marimo/_server/model.py@@ -1,9 +1,8 @@# Copyright 2024 Marimo. All rights reserved.from __future__ import annotations-import abcfrom enum import Enum-from typing import TYPE_CHECKING, Callable+from typing import TYPE_CHECKING, Callable, Protocolfrom marimo._types.ids import ConsumerId@@ -30,7 +29,7 @@ class SessionMode(str, Enum):RUN = "run"-class SessionConsumer(metaclass=abc.ABCMeta):+class SessionConsumer(Protocol):"""Consumer for a session@@ -38,10 +37,9 @@ class SessionConsumer(metaclass=abc.ABCMeta):connection types. Currently we consume a session via WebSocket"""- def __init__(self, consumer_id: ConsumerId) -> None:- self.consumer_id = consumer_id+ @property+ def consumer_id(self) -> ConsumerId: ...- @abc.abstractmethoddef on_start(self,) -> Callable[[KernelMessage], None]:@@ -49,16 +47,10 @@ class SessionConsumer(metaclass=abc.ABCMeta):Start the session consumerand return a subscription function for the session consumer"""- raise NotImplementedError+ ...- @abc.abstractmethod- def on_stop(self) -> None:- raise NotImplementedError+ def on_stop(self) -> None: ...- @abc.abstractmethod- def write_operation(self, op: MessageOperation) -> None:- raise NotImplementedError+ def write_operation(self, op: MessageOperation) -> None: ...- @abc.abstractmethod- def connection_state(self) -> ConnectionState:- raise NotImplementedError+ def connection_state(self) -> ConnectionState: ...diff --git a/marimo/_server/notebook/storage.py b/marimo/_server/notebook/storage.pyindex 9e344edd77..4fa8625575 100644--- a/marimo/_server/notebook/storage.py+++ b/marimo/_server/notebook/storage.py@@ -3,9 +3,8 @@from __future__ import annotations-from abc import ABC, abstractmethodfrom pathlib import Path-from typing import Optional+from typing import Optional, Protocolfrom marimo import _loggersfrom marimo._server.api.status import HTTPException, HTTPStatus@@ -13,10 +12,9 @@ from marimo._server.api.status import HTTPException, HTTPStatusLOGGER = _loggers.marimo_logger()-class StorageInterface(ABC):- """Abstract interface for storage operations."""+class StorageInterface(Protocol):+ """Interface for storage operations."""- @abstractmethoddef read(self, path: Path) -> str:"""Read content from storage.@@ -29,9 +27,8 @@ class StorageInterface(ABC):Raises:HTTPException: If read fails"""- pass+ ...- @abstractmethoddef write(self, path: Path, content: str) -> None:"""Write content to storage.@@ -42,9 +39,8 @@ class StorageInterface(ABC):Raises:HTTPException: If write fails"""- pass+ ...- @abstractmethoddef exists(self, path: Path) -> bool:"""Check if path exists in storage.@@ -54,9 +50,8 @@ class StorageInterface(ABC):Returns:True if path exists, False otherwise"""- pass+ ...- @abstractmethoddef rename(self, old_path: Path, new_path: Path) -> None:"""Rename/move a file in storage.@@ -67,9 +62,8 @@ class StorageInterface(ABC):Raises:HTTPException: If rename fails"""- pass+ ...- @abstractmethoddef is_same_path(self, path1: Path, path2: Path) -> bool:"""Check if two paths refer to the same location.@@ -80,9 +74,8 @@ class StorageInterface(ABC):Returns:True if paths refer to same location"""- pass+ ...- @abstractmethoddef get_absolute_path(self, path: Path) -> Path:"""Get absolute path.@@ -92,9 +85,8 @@ class StorageInterface(ABC):Returns:Absolute path"""- pass+ ...- @abstractmethoddef read_related_file(self, base_path: Path, relative_path: str) -> Optional[str]:@@ -109,7 +101,7 @@ class StorageInterface(ABC):Returns:File contents or None if not found"""- pass+ ...class FilesystemStorage(StorageInterface):diff --git a/marimo/_server/sessions/managers.py b/marimo/_server/sessions/managers.pyindex 33eec9aa2d..acf0053b1d 100644--- a/marimo/_server/sessions/managers.py+++ b/marimo/_server/sessions/managers.py@@ -228,6 +228,15 @@ class KernelManagerImpl(KernelManager):listener.accept())+ @property+ def pid(self) -> int | None:+ """Get the PID of the kernel."""+ if self.kernel_task is None:+ return None+ if isinstance(self.kernel_task, threading.Thread):+ return None+ return self.kernel_task.pid+@propertydef profile_path(self) -> str | None:self._profile_path: str | Nonediff --git a/marimo/_server/sessions/session.py b/marimo/_server/sessions/session.pyindex cbffcd912a..f8b69f3566 100644--- a/marimo/_server/sessions/session.py+++ b/marimo/_server/sessions/session.py@@ -43,17 +43,23 @@ from marimo._server.sessions.managers import (QueueManagerImpl,)from marimo._server.sessions.room import Room-from marimo._server.sessions.types import KernelManager, QueueManager, Session+from marimo._server.sessions.types import (+ KernelManager,+ KernelState,+ QueueManager,+ Session,+)from marimo._server.utils import print_, print_tabbedfrom marimo._types.ids import ConsumerIdfrom marimo._utils.distributor import (ConnectionDistributor,+ Distributor,QueueDistributor,)from marimo._utils.repr import format_reprif TYPE_CHECKING:- from collections.abc import Mapping+ from collections.abc import Callable, MappingLOGGER = _loggers.marimo_logger()@@ -62,6 +68,74 @@ _DEFAULT_TTL_SECONDS = 120__all__ = ["Session", "SessionImpl"]+class SessionHeartbeatMonitor:+ """Monitors kernel health and handles cleanup when kernel dies."""++ def __init__(+ self,+ kernel_manager: KernelManager,+ on_kernel_died: Callable[[], None],+ app_filename: Optional[str],+ ) -> None:+ """Initialize the heartbeat monitor.++ Args:+ kernel_manager: The kernel manager to monitor+ on_kernel_died: Callback to invoke when kernel dies+ app_filename: Filename for error messages (optional)+ """+ self.kernel_manager = kernel_manager+ self.on_kernel_died = on_kernel_died+ self.app_filename = app_filename+ self.heartbeat_task: Optional[asyncio.Task[Any]] = None++ def start(self) -> None:+ """Start the heartbeat monitoring."""++ def _check_alive() -> None:+ if not self.kernel_manager.is_alive():+ LOGGER.debug("Kernel died, invoking cleanup callback")+ self.on_kernel_died()+ print_()+ filename_str = self.app_filename or "unknown"+ print_tabbed(+ red(+ "The Python kernel for file "+ f"{filename_str} died unexpectedly."+ )+ )+ print_()++ async def _heartbeat() -> None:+ while True:+ await asyncio.sleep(1)+ _check_alive()++ try:+ loop = asyncio.get_event_loop()+ self.heartbeat_task = loop.create_task(_heartbeat())+ except RuntimeError:+ # This can happen if there is no event loop running+ self.heartbeat_task = None++ def stop(self) -> None:+ """Stop the heartbeat monitoring."""+ if self.heartbeat_task:+ self.heartbeat_task.cancel()+++def _create_message_distributor(+ kernel_manager: KernelManager,+ queue_manager: QueueManager,+) -> Distributor[KernelMessage]:+ if kernel_manager.mode == SessionMode.EDIT:+ return ConnectionDistributor(kernel_manager.kernel_connection)+ else:+ q = queue_manager.stream_queue+ assert q is not None+ return QueueDistributor(queue=q)++class SessionImpl(Session):"""A client session.@@ -145,20 +219,17 @@ class SessionImpl(Session):self.session_cache_manager: SessionCacheManager | None = Noneself.config_manager = config_managerself.kernel_manager.start_kernel()- # Reads from the kernel connection and distributes the- # messages to each subscriber.- self.message_distributor: (- ConnectionDistributor[KernelMessage]- | QueueDistributor[KernelMessage]++ # Create composed objects+ self.message_distributor = _create_message_distributor(+ kernel_manager=self.kernel_manager,+ queue_manager=queue_manager,+ )+ self.heartbeat_monitor = SessionHeartbeatMonitor(+ kernel_manager=self.kernel_manager,+ on_kernel_died=self._on_kernel_died,+ app_filename=self.app_file_manager.filename,)- if self.kernel_manager.mode == SessionMode.EDIT:- self.message_distributor = ConnectionDistributor[KernelMessage](- self.kernel_manager.kernel_connection- )- else:- q = self._queue_manager.stream_queue- assert q is not None- self.message_distributor = QueueDistributor[KernelMessage](queue=q)self.message_distributor.add_consumer(lambda msg: self.session_view.add_raw_operation(msg)@@ -166,8 +237,7 @@ class SessionImpl(Session):self.connect_consumer(session_consumer, main=True)self.message_distributor.start()- self.heartbeat_task: Optional[asyncio.Task[Any]] = None- self._start_heartbeat()+ self.heartbeat_monitor.start()self._closed = False@property@@ -175,43 +245,39 @@ class SessionImpl(Session):"""Get the consumers in the session."""return self.room.consumers- def _start_heartbeat(self) -> None:- def _check_alive() -> None:- if not self.kernel_manager.is_alive():- LOGGER.debug(- "Closing session %s because kernel died",- self.initialization_id,- )- self.close()- print_()- print_tabbed(- red(- "The Python kernel for file "- f"{self.app_file_manager.filename} died unexpectedly."- )- )- print_()- self.close()-- # Start a heartbeat task, which checks if the kernel is alive- # every second+ def flush_messages(self) -> None:+ """Flush any pending messages."""+ self.message_distributor.flush()- async def _heartbeat() -> None:- while True:- await asyncio.sleep(1)- _check_alive()+ def rename_path(self, new_path: str) -> None:+ """Rename the path of the session."""+ if self.session_cache_manager:+ self.session_cache_manager.rename_path(new_path)- try:- loop = asyncio.get_event_loop()- self.heartbeat_task = loop.create_task(_heartbeat())- except RuntimeError:- # This can happen if there is no event loop running- self.heartbeat_task = None+ def _on_kernel_died(self) -> None:+ """Callback invoked when the kernel dies unexpectedly."""+ LOGGER.debug(+ "Closing session %s because kernel died",+ self.initialization_id,+ )+ self.close()def try_interrupt(self) -> None:"""Try to interrupt the kernel."""self.kernel_manager.interrupt_kernel()+ def kernel_state(self) -> KernelState:+ """Get the state of the kernel."""+ if self.kernel_manager.kernel_task is None:+ return KernelState.NOT_STARTED+ if self.kernel_manager.kernel_task.is_alive():+ return KernelState.RUNNING+ return KernelState.STOPPED++ def kernel_pid(self) -> int | None:+ """Get the PID of the kernel."""+ return self.kernel_manager.pid+def put_control_request(self,request: requests.ControlRequest,@@ -326,8 +392,7 @@ class SessionImpl(Session):self.room.close()# Close the kernelself.message_distributor.stop()- if self.heartbeat_task:- self.heartbeat_task.cancel()+ self.heartbeat_monitor.stop()if self.session_cache_manager:self.session_cache_manager.stop()self.kernel_manager.close_kernel()diff --git a/marimo/_server/sessions/session_manager.py b/marimo/_server/sessions/session_manager.pyindex 38729b3254..6b0a0e38f5 100644--- a/marimo/_server/sessions/session_manager.py+++ b/marimo/_server/sessions/session_manager.py@@ -38,6 +38,7 @@ from marimo._server.sessions.resume_strategies import create_resume_strategyfrom marimo._server.sessions.session import Session, SessionImplfrom marimo._server.sessions.session_repository import SessionRepositoryfrom marimo._server.sessions.token_manager import TokenManager+from marimo._server.sessions.types import KernelStatefrom marimo._server.tokens import AuthToken, SkewProtectionTokenfrom marimo._types.ids import ConsumerId, SessionIdfrom marimo._utils.file_watcher import FileWatcherManager@@ -238,9 +239,7 @@ class SessionManager:if not session.app_file_manager.path:return False, "Session has no associated file"- # Handle rename for session cache- if session.session_cache_manager:- session.session_cache_manager.rename_path(new_path)+ session.rename_path(new_path)try:if self.watch and self._file_watcher_lifecycle:@@ -329,8 +328,7 @@ class SessionManager:for session_id in list(self._repository.get_all_session_ids()):session = self._repository.get_sync(session_id)if session:- task = session.kernel_manager.kernel_task- if task is not None and not task.is_alive():+ if session.kernel_state() is KernelState.STOPPED:self.close_session(session_id)def any_clients_connected(self, key: MarimoFileKey) -> bool:diff --git a/marimo/_server/sessions/types.py b/marimo/_server/sessions/types.pyindex 6840e562ba..fbc7490b9d 100644--- a/marimo/_server/sessions/types.py+++ b/marimo/_server/sessions/types.py@@ -8,6 +8,7 @@ implementations.from __future__ import annotations+from enum import Enumfrom typing import TYPE_CHECKING, Any, Optional, Protocol, Unionfrom marimo._messaging.ops import MessageOperation@@ -25,11 +26,6 @@ if TYPE_CHECKING:from collections.abc import Mappingfrom marimo._config.manager import MarimoConfigManager- from marimo._server.session.serialize import SessionCacheManager- from marimo._utils.distributor import (- ConnectionDistributor,- QueueDistributor,- )class QueueManager(Protocol):@@ -69,6 +65,11 @@ class KernelManager(Protocol):"""Close the kernel and clean up resources."""...+ @property+ def pid(self) -> int | None:+ """Get the PID of the kernel."""+ ...+@propertydef kernel_connection(self) -> TypedConnection[KernelMessage]:"""Get the kernel connection for reading messages."""@@ -80,6 +81,14 @@ class KernelManager(Protocol):...+class KernelState(Enum):+ """Kernel state."""++ NOT_STARTED = "not_started"+ RUNNING = "running"+ STOPPED = "stopped"++class Session(Protocol):"""Protocol for session management."""@@ -87,14 +96,6 @@ class Session(Protocol):session_view: SessionViewapp_file_manager: AppFileManagerconfig_manager: MarimoConfigManager-- # TODO: don't expose these internals- session_cache_manager: Optional[SessionCacheManager]- message_distributor: Union[- ConnectionDistributor[KernelMessage],- QueueDistributor[KernelMessage],- ]- kernel_manager: KernelManagerttl_seconds: int@property@@ -102,10 +103,26 @@ class Session(Protocol):"""Get the consumers in the session."""...+ def kernel_state(self) -> KernelState:+ """Get the state of the kernel."""+ ...++ def kernel_pid(self) -> int | None:+ """Get the PID of the kernel."""+ ...+def try_interrupt(self) -> None:"""Try to interrupt the kernel."""...+ def flush_messages(self) -> None:+ """Flush any pending messages."""+ ...++ def rename_path(self, new_path: str) -> None:+ """Rename the path of the session."""+ ...+def put_control_request(self,request: requests.ControlRequest,diff --git a/marimo/_utils/distributor.py b/marimo/_utils/distributor.pyindex e4824c5633..1feffaaefb 100644--- a/marimo/_utils/distributor.py+++ b/marimo/_utils/distributor.py@@ -4,7 +4,7 @@ from __future__ import annotationsimport asyncioimport threadingimport time-from typing import Callable, Generic, TypeVar, Union+from typing import Any, Callable, Generic, Protocol, TypeVar, Unionfrom marimo import _loggersfrom marimo._server.types import QueueType@@ -13,13 +13,33 @@ from marimo._utils.typed_connection import TypedConnectionLOGGER = _loggers.marimo_logger()-T = TypeVar("T")+T = TypeVar("T", covariant=True)Consumer = Callable[[T], None]-class ConnectionDistributor(Generic[T]):+class Distributor(Generic[T], Protocol):+ """Base class for distributors."""++ def add_consumer(self, consumer: Consumer[T]) -> Disposable:+ """Add a consumer to the distributor."""+ ...++ def start(self) -> Any:+ """Start the distributor."""+ ...++ def stop(self) -> None:+ """Stop the distributor."""+ ...++ def flush(self) -> None:+ """Flush the distributor."""+ ...+++class ConnectionDistributor(Distributor[T]):"""Used to distribute the response of a multiprocessing Connection to multipleconsumers.@@ -92,7 +112,7 @@ class ConnectionDistributor(Generic[T]):break-class QueueDistributor(Generic[T]):+class QueueDistributor(Distributor[T]):def __init__(self, queue: QueueType[Union[T, None]]) -> None:self.consumers: list[Consumer[T]] = []# distributor uses None as a signal to stop