A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat: implement kernel exit classification and notification system (#9500)
Michael Chavinda59ba8407ee64
diff --git a/marimo/_session/extensions/extensions.py b/marimo/_session/extensions/extensions.pyindex 648251f684..f9194fb073 100644--- a/marimo/_session/extensions/extensions.py+++ b/marimo/_session/extensions/extensions.py@@ -22,6 +22,7 @@ from marimo._messaging.notebook.changes import TransactionSourcefrom marimo._messaging.notebook.document import NotebookCellfrom marimo._messaging.notification import (AlertNotification,+ BannerNotification,NotebookDocumentTransactionNotification,NotificationMessage,)@@ -87,25 +88,60 @@ class HeartbeatExtension(SessionExtension):def _start(self, session: Session) -> None:"""Start the heartbeat monitoring."""- def _check_alive() -> None:- if session.kernel_state() == KernelState.STOPPED:+ async def _check_alive() -> None:+ # Outer guard: an unhandled exception here would kill the+ # heartbeat task and we'd never detect future kernel deaths in+ # this session's lifetime. CancelledError is re-raised so detach+ # still cleanly stops the task.+ try:+ if session.kernel_state() != KernelState.STOPPED:+ return+ exit_info = session.kernel_exit_info()LOGGER.debug("Kernel died, invoking cleanup callback")+ reason = (+ exit_info.message if exit_info is not None else "unknown"+ )+ # Notify the frontend before closing the WS so the user sees+ # a persistent banner with the real cause instead of just a+ # "disconnected" UI. ``notify`` only queues the frame on each+ # consumer's send queue; yield to the event loop afterwards+ # so the WS writer task drains it before ``session.close``+ # detaches the consumers. Inner guard isolates a broadcast+ # failure from cleanup -- we still want to close the session+ # and log even if the banner can't be delivered.+ try:+ session.notify(+ BannerNotification(+ title="Kernel stopped",+ description=html.escape(reason),+ variant="danger",+ action="restart",+ ),+ from_consumer_id=None,+ )+ await asyncio.sleep(0.1)+ except Exception:+ LOGGER.exception("Failed to broadcast kernel-died banner")session.close()print_()filename = session.app_file_manager.filenamefilename_str = filename or "unknown"print_tabbed(red(- "The Python kernel for file "- f"{filename_str} died unexpectedly."+ f"The Python kernel for file {filename_str} died: "+ f"{reason}"))print_()+ except asyncio.CancelledError:+ raise+ except Exception:+ LOGGER.exception("Unexpected error in kernel heartbeat check")async def _heartbeat() -> None:while True:await asyncio.sleep(1)- _check_alive()+ await _check_alive()try:loop = asyncio.get_event_loop()diff --git a/marimo/_session/kernel_exit.py b/marimo/_session/kernel_exit.pynew file mode 100644index 0000000000..23946f0c24--- /dev/null+++ b/marimo/_session/kernel_exit.py@@ -0,0 +1,186 @@+# Copyright 2026 Marimo. All rights reserved.+"""Classify how a kernel subprocess exited.++Used to turn a raw ``multiprocessing.Process.exitcode`` into a structured+``KernelExitInfo`` with a human-readable message. Best-effort cgroup+inspection adds memory diagnostics on Linux when the kernel was SIGKILLed,+which on a container is almost always the cgroup OOM killer.+"""++from __future__ import annotations++import os+import signal+import sys++from marimo._session.types import KernelExitInfo++# POSIX signal numbers used for kernel-exit classification. We pull them from+# the ``signal`` module so the branches read naturally, with numeric fallbacks+# for Windows (which doesn't define SIGKILL). The surrounding code is gated to+# Linux at runtime, so the fallbacks only matter for tests that monkeypatch+# ``sys.platform``.+_SIGKILL = getattr(signal, "SIGKILL", 9)+_SIGSEGV = getattr(signal, "SIGSEGV", 11)++# Set by the container runtime / launcher to the pod's memory limit in+# mebibytes (binary, 1 MiB = 1024 * 1024 bytes). Used purely to enrich OOM+# messages; safe to leave unset.+_MEMORY_LIMIT_ENV = "MARIMO_KERNEL_MEMORY_LIMIT_MIB"++# cgroup v1 paths (used in molab's gVisor sandbox today). v2 uses different+# filenames under /sys/fs/cgroup directly; we try both.+_CGROUP_V1_FAILCNT = "/sys/fs/cgroup/memory/memory.failcnt"+_CGROUP_V1_PEAK = "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"+_CGROUP_V2_EVENTS = "/sys/fs/cgroup/memory.events"+_CGROUP_V2_PEAK = "/sys/fs/cgroup/memory.peak"+++_RESTART_HINT = "Click Restart to start a fresh kernel."+++def classify_kernel_exit(exitcode: int | None) -> KernelExitInfo:+ """Return a KernelExitInfo describing how the kernel exited.++ Messages are end-user-facing: each one names the cause in plain language+ and ends with a concrete next step (typically pressing Restart, sometimes+ paired with advice on freeing memory or trimming inputs).+ """+ if exitcode is None:+ return KernelExitInfo(+ exitcode=None,+ cause="unknown",+ message=f"The kernel stopped unexpectedly. {_RESTART_HINT}",+ )+ if exitcode == 0:+ return KernelExitInfo(+ exitcode=0,+ cause="success",+ message=f"The kernel finished cleanly. {_RESTART_HINT}",+ )+ if exitcode > 0:+ return KernelExitInfo(+ exitcode=exitcode,+ cause="exit",+ message=(+ f"The kernel exited with error code {exitcode}. "+ f"{_RESTART_HINT}"+ ),+ )++ # ``exitcode < 0`` only occurs on POSIX (Windows never returns negative+ # exitcodes). The signal-name and cgroup OOM logic below assumes Linux+ # conventions (POSIX signal numbers per multiprocessing.Process.exitcode,+ # see https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Process.exitcode,+ # and Linux-specific cgroup files under /sys/fs/cgroup). On other+ # platforms (including darwin) we have no way to confirm OOM, so we+ # surface a generic failure rather than guessing -- claiming OOM for+ # every SIGKILL would mislead users when the real cause is a crash,+ # ``kill -9``, or something else.+ if sys.platform != "linux":+ return KernelExitInfo(+ exitcode=exitcode,+ cause="abnormal",+ message=(f"The kernel was stopped unexpectedly. {_RESTART_HINT}"),+ )++ signal_num = -exitcode+ if signal_num == _SIGKILL:+ peak_mib = _peak_rss_mib()+ if _was_oom_killed():+ limit_mib = _memory_limit_mib()+ usage = (+ f" Notebook used {peak_mib} MiB of the {limit_mib} MiB limit."+ if peak_mib is not None and limit_mib is not None+ else f" Peak memory: {peak_mib} MiB."+ if peak_mib is not None+ else ""+ )+ return KernelExitInfo(+ exitcode=exitcode,+ cause="oom",+ message=(+ "The kernel ran out of memory and was stopped."+ + usage+ + " Free large variables (e.g. `del x; import gc; "+ "gc.collect()`), reduce data sizes, or request a larger "+ f"sandbox. {_RESTART_HINT}"+ ),+ )+ # SIGKILL without cgroup-confirmed OOM: the cause is genuinely+ # unknown (could be ``kill -9``, an external OOM-killer outside our+ # cgroup, a crash, etc.). Be honest rather than guessing OOM.+ return KernelExitInfo(+ exitcode=exitcode,+ cause="sigkill",+ message=(f"The kernel was stopped unexpectedly. {_RESTART_HINT}"),+ )+ elif signal_num == _SIGSEGV:+ return KernelExitInfo(+ exitcode=exitcode,+ cause="segfault",+ message=(+ "A native extension crashed the kernel (segmentation "+ f"fault). {_RESTART_HINT} If it recurs, try a different "+ "version of the package that triggered it."+ ),+ )+ return KernelExitInfo(+ exitcode=exitcode,+ cause=f"signal_{signal_num}",+ message=(+ f"The kernel was stopped by the operating system (signal "+ f"{signal_num}). {_RESTART_HINT}"+ ),+ )+++def _was_oom_killed() -> bool:+ failcnt = _read_int(_CGROUP_V1_FAILCNT)+ if failcnt is not None:+ return failcnt > 0+ oom_kill = _read_kv(_CGROUP_V2_EVENTS, "oom_kill")+ if oom_kill is not None:+ return oom_kill > 0+ return False+++def _peak_rss_mib() -> int | None:+ for path in (_CGROUP_V1_PEAK, _CGROUP_V2_PEAK):+ value = _read_int(path)+ if value is not None:+ return value // (1024 * 1024)+ return None+++def _memory_limit_mib() -> int | None:+ raw = os.environ.get(_MEMORY_LIMIT_ENV)+ if raw is None:+ return None+ try:+ return int(raw)+ except ValueError:+ return None+++def _read_int(path: str) -> int | None:+ try:+ with open(path, encoding="utf-8") as f:+ return int(f.read().strip())+ except (OSError, ValueError):+ return None+++def _read_kv(path: str, key: str) -> int | None:+ try:+ with open(path, encoding="utf-8") as f:+ for line in f:+ parts = line.split()+ if len(parts) == 2 and parts[0] == key:+ try:+ return int(parts[1])+ except ValueError:+ return None+ except OSError:+ return None+ return Nonediff --git a/marimo/_session/managers/ipc.py b/marimo/_session/managers/ipc.pyindex 69c8d0d664..9510401078 100644--- a/marimo/_session/managers/ipc.py+++ b/marimo/_session/managers/ipc.py@@ -427,6 +427,11 @@ class _SubprocessWrapper(ProcessLike):def pid(self) -> int | None:return self._process.pid+ @property+ def exitcode(self) -> int | None:+ """Mirror multiprocessing.Process.exitcode for exit diagnostics."""+ return self._process.poll()+def is_alive(self) -> bool:return self._process.poll() is Nonediff --git a/marimo/_session/session.py b/marimo/_session/session.pyindex 6e4a88992d..eedc216595 100644--- a/marimo/_session/session.py+++ b/marimo/_session/session.py@@ -45,6 +45,7 @@ from marimo._session.extensions.types import (ExtensionRegistry,SessionExtension,)+from marimo._session.kernel_exit import classify_kernel_exitfrom marimo._session.managers import (KernelManagerImpl,QueueManagerImpl,@@ -54,6 +55,7 @@ from marimo._session.notebook import AppFileManagerfrom marimo._session.room import Roomfrom marimo._session.state.session_view import SessionViewfrom marimo._session.types import (+ KernelExitInfo,KernelManager,KernelState,QueueManager,@@ -338,6 +340,16 @@ class SessionImpl(Session):"""Get the PID of the kernel."""return self._kernel_manager.pid+ def kernel_exit_info(self) -> KernelExitInfo | None:+ """Describe how the kernel exited."""+ task = self._kernel_manager.kernel_task+ if task is None or task.is_alive():+ return None+ # ``exitcode`` is provided by multiprocessing.Process; threads don't+ # have one, so we treat absence as "unknown".+ exitcode = getattr(task, "exitcode", None)+ return classify_kernel_exit(exitcode)+def put_control_request(self,request: commands.CommandMessage,diff --git a/marimo/_session/types.py b/marimo/_session/types.pyindex 6df5b36490..641544730a 100644--- a/marimo/_session/types.py+++ b/marimo/_session/types.py@@ -9,6 +9,7 @@ implementations.from __future__ import annotationsimport contextlib+from dataclasses import dataclassfrom enum import Enumfrom typing import TYPE_CHECKING, Any, Protocol@@ -104,6 +105,25 @@ class KernelState(Enum):STOPPED = "stopped"+@dataclass(frozen=True)+class KernelExitInfo:+ """Information about how a kernel exited.++ Populated after the kernel task has stopped. ``exitcode`` follows the+ convention of ``multiprocessing.Process.exitcode``: ``>= 0`` for a normal+ exit with that status, and ``< 0`` if the process was terminated by signal+ ``-exitcode``. ``None`` means the exit status is unavailable -- either the+ task has not yet terminated, or the underlying task type does not expose+ one (e.g. threads). ``cause`` is a short machine-readable tag and+ ``message`` is a human-readable one-liner suitable for logs or end-user+ display.+ """++ exitcode: int | None+ cause: str+ message: str++class Session(Protocol):"""Protocol for session management."""@@ -132,6 +152,13 @@ class Session(Protocol):"""Get the PID of the kernel."""...+ def kernel_exit_info(self) -> KernelExitInfo | None:+ """Describe how the kernel exited, or ``None`` if it is still running.++ Only meaningful once ``kernel_state() == KernelState.STOPPED``.+ """+ ...+def try_interrupt(self) -> None:"""Try to interrupt the kernel."""...diff --git a/tests/_session/extensions/test_extensions.py b/tests/_session/extensions/test_extensions.pyindex 3d0cc1a5d3..451a2f7168 100644--- a/tests/_session/extensions/test_extensions.py+++ b/tests/_session/extensions/test_extensions.py@@ -8,6 +8,7 @@ from unittest.mock import Mock, patchimport pytest+from marimo._messaging.notification import BannerNotificationfrom marimo._session.events import SessionEventBusfrom marimo._session.extensions.extensions import (CacheMode,@@ -24,7 +25,7 @@ from marimo._session.extensions.types import (ExtensionRegistry,)from marimo._session.model import ConnectionState, SessionMode-from marimo._session.types import KernelState+from marimo._session.types import KernelExitInfo, KernelStatefrom marimo._types.ids import CellId_t, RequestId, SessionId@@ -74,12 +75,30 @@ class TestHeartbeatExtension:async def test_detects_dead_kernel(self, mock_session, event_bus) -> None:"""Test that heartbeat detects when kernel dies and closes session."""mock_session.kernel_state.return_value = KernelState.STOPPED+ mock_session.kernel_exit_info.return_value = KernelExitInfo(+ exitcode=-9,+ cause="oom",+ message=(+ "The kernel ran out of memory and was stopped. "+ "Notebook used 1952 MiB of the 2048 MiB limit. "+ "Click Restart to start a fresh kernel."+ ),+ )extension = HeartbeatExtension()extension.on_attach(mock_session, event_bus)await asyncio.sleep(1.5)mock_session.close.assert_called_once()+ # A persistent banner is broadcast before the session closes, so the+ # frontend can show the cause instead of just a "disconnected" toast.+ mock_session.notify.assert_called_once()+ banner = mock_session.notify.call_args.args[0]+ assert isinstance(banner, BannerNotification)+ assert banner.variant == "danger"+ assert banner.action == "restart"+ assert "out of memory" in banner.description+ assert "restart" in banner.description.lower()extension.on_detach()diff --git a/tests/_session/managers/test_ipc.py b/tests/_session/managers/test_ipc.pyindex 734a05fe6a..07ef5672fe 100644--- a/tests/_session/managers/test_ipc.py+++ b/tests/_session/managers/test_ipc.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport osimport timefrom pathlib import Path+from unittest.mock import MagicMockimport pytest@@ -107,6 +108,17 @@ class TestIPCKernelManagerImpl:assert kernel_manager.venv_python == "/path/to/sandbox/venv/python"+class TestSubprocessWrapper:+ def test_exitcode_uses_popen_returncode(self) -> None:+ from marimo._session.managers.ipc import _SubprocessWrapper++ process = MagicMock()+ process.poll.return_value = -9+ wrapper = _SubprocessWrapper(process)++ assert wrapper.exitcode == -9++@pytest.mark.requires("zmq")class TestIPCQueueManagerImpl:def test_from_ipc_factory(self) -> None:diff --git a/tests/_session/test_kernel_exit.py b/tests/_session/test_kernel_exit.pynew file mode 100644index 0000000000..568ad0b5aa--- /dev/null+++ b/tests/_session/test_kernel_exit.py@@ -0,0 +1,135 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++import sys+from typing import TYPE_CHECKING++from marimo._session import kernel_exit+from marimo._session.kernel_exit import classify_kernel_exit++if TYPE_CHECKING:+ import pytest+++def _force_linux(monkeypatch: pytest.MonkeyPatch) -> None:+ """Pin sys.platform so signal/cgroup classification is exercised on any host."""+ monkeypatch.setattr(sys, "platform", "linux")+++def test_unknown_when_exitcode_none() -> None:+ info = classify_kernel_exit(None)+ assert info.cause == "unknown"+ assert info.exitcode is None+++def test_success_exit() -> None:+ info = classify_kernel_exit(0)+ assert info.cause == "success"+ assert info.exitcode == 0+++def test_nonzero_exit() -> None:+ info = classify_kernel_exit(2)+ assert info.cause == "exit"+ assert "code 2" in info.message+++def test_segfault_on_linux(monkeypatch: pytest.MonkeyPatch) -> None:+ _force_linux(monkeypatch)+ info = classify_kernel_exit(-11)+ assert info.cause == "segfault"+ assert "segmentation" in info.message.lower()+++def test_sigkill_without_oom_signal(monkeypatch: pytest.MonkeyPatch) -> None:+ # No cgroup files readable -> SIGKILL with no OOM evidence.+ # We must NOT claim OOM since the real cause could be a manual kill,+ # an external OOM-killer outside our cgroup, a crash, etc.+ _force_linux(monkeypatch)+ monkeypatch.setattr(kernel_exit, "_read_int", lambda _path: None)+ monkeypatch.setattr(kernel_exit, "_read_kv", lambda _path, _key: None)+ info = classify_kernel_exit(-9)+ assert info.cause == "sigkill"+ assert "out of memory" not in info.message.lower()+ assert "restart" in info.message.lower()+++def test_sigkill_with_cgroup_oom_v1(monkeypatch: pytest.MonkeyPatch) -> None:+ _force_linux(monkeypatch)++ def fake_read_int(path: str) -> int | None:+ if "failcnt" in path:+ return 5+ if "max_usage" in path:+ return 1952 * 1024 * 1024+ return None++ monkeypatch.setattr(kernel_exit, "_read_int", fake_read_int)+ monkeypatch.setattr(kernel_exit, "_read_kv", lambda _path, _key: None)+ monkeypatch.setenv(kernel_exit._MEMORY_LIMIT_ENV, "2048")++ info = classify_kernel_exit(-9)+ assert info.cause == "oom"+ assert "1952 MiB" in info.message+ assert "2048 MiB" in info.message+++def test_sigkill_with_cgroup_oom_v2(monkeypatch: pytest.MonkeyPatch) -> None:+ _force_linux(monkeypatch)++ def fake_read_int(path: str) -> int | None:+ if path == kernel_exit._CGROUP_V2_PEAK:+ return 100 * 1024 * 1024+ return None++ def fake_read_kv(path: str, key: str) -> int | None:+ if path == kernel_exit._CGROUP_V2_EVENTS and key == "oom_kill":+ return 1+ return None++ monkeypatch.setattr(kernel_exit, "_read_int", fake_read_int)+ monkeypatch.setattr(kernel_exit, "_read_kv", fake_read_kv)+ monkeypatch.delenv(kernel_exit._MEMORY_LIMIT_ENV, raising=False)++ info = classify_kernel_exit(-9)+ assert info.cause == "oom"+ assert "100 MiB" in info.message+++def test_other_signal_on_linux(monkeypatch: pytest.MonkeyPatch) -> None:+ _force_linux(monkeypatch)+ info = classify_kernel_exit(-15) # SIGTERM+ assert info.cause == "signal_15"+ assert "signal 15" in info.message+ assert "restart" in info.message.lower()+++def test_negative_exitcode_on_non_linux(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # On Windows/macOS we have no way to confirm OOM; surface a generic+ # failure rather than guessing. Don't leak raw signal jargon either.+ monkeypatch.setattr(sys, "platform", "win32")+ info = classify_kernel_exit(-9)+ assert info.cause == "abnormal"+ assert "out of memory" not in info.message.lower()+ assert "restart" in info.message.lower()+ assert "SIGKILL" not in info.message+ assert "exitcode" not in info.message.lower()++ monkeypatch.setattr(sys, "platform", "darwin")+ info = classify_kernel_exit(-11)+ assert info.cause == "abnormal"+ assert "out of memory" not in info.message.lower()+ assert "restart" in info.message.lower()+++def test_positive_exitcode_on_non_linux(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Positive exitcodes still classify the same way on every platform.+ monkeypatch.setattr(sys, "platform", "win32")+ info = classify_kernel_exit(1)+ assert info.cause == "exit"+ assert "code 1" in info.message+ assert "restart" in info.message.lower()