Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat: implement kernel exit classification and notification system (#9500)

Michael Chavinda59ba8407ee64
diff --git a/marimo/_session/extensions/extensions.py b/marimo/_session/extensions/extensions.py
index 648251f684..f9194fb073 100644
--- a/marimo/_session/extensions/extensions.py
+++ b/marimo/_session/extensions/extensions.py
@@ -22,6 +22,7 @@ from marimo._messaging.notebook.changes import TransactionSource
from marimo._messaging.notebook.document import NotebookCell
from marimo._messaging.notification import (
AlertNotification,
+ BannerNotification,
NotebookDocumentTransactionNotification,
NotificationMessage,
)
@@ -87,25 +88,60 @@ class HeartbeatExtension(SessionExtension):
def _start(self, session: Session) -> None:
"""Start the heartbeat monitoring."""
- def _check_alive() -> None:
- if session.kernel_state() == KernelState.STOPPED:
+ async def _check_alive() -> None:
+ # Outer guard: an unhandled exception here would kill the
+ # heartbeat task and we'd never detect future kernel deaths in
+ # this session's lifetime. CancelledError is re-raised so detach
+ # still cleanly stops the task.
+ try:
+ if session.kernel_state() != KernelState.STOPPED:
+ return
+ exit_info = session.kernel_exit_info()
LOGGER.debug("Kernel died, invoking cleanup callback")
+ reason = (
+ exit_info.message if exit_info is not None else "unknown"
+ )
+ # Notify the frontend before closing the WS so the user sees
+ # a persistent banner with the real cause instead of just a
+ # "disconnected" UI. ``notify`` only queues the frame on each
+ # consumer's send queue; yield to the event loop afterwards
+ # so the WS writer task drains it before ``session.close``
+ # detaches the consumers. Inner guard isolates a broadcast
+ # failure from cleanup -- we still want to close the session
+ # and log even if the banner can't be delivered.
+ try:
+ session.notify(
+ BannerNotification(
+ title="Kernel stopped",
+ description=html.escape(reason),
+ variant="danger",
+ action="restart",
+ ),
+ from_consumer_id=None,
+ )
+ await asyncio.sleep(0.1)
+ except Exception:
+ LOGGER.exception("Failed to broadcast kernel-died banner")
session.close()
print_()
filename = session.app_file_manager.filename
filename_str = filename or "unknown"
print_tabbed(
red(
- "The Python kernel for file "
- f"{filename_str} died unexpectedly."
+ f"The Python kernel for file {filename_str} died: "
+ f"{reason}"
)
)
print_()
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ LOGGER.exception("Unexpected error in kernel heartbeat check")
async def _heartbeat() -> None:
while True:
await asyncio.sleep(1)
- _check_alive()
+ await _check_alive()
try:
loop = asyncio.get_event_loop()
diff --git a/marimo/_session/kernel_exit.py b/marimo/_session/kernel_exit.py
new file mode 100644
index 0000000000..23946f0c24
--- /dev/null
+++ b/marimo/_session/kernel_exit.py
@@ -0,0 +1,186 @@
+# Copyright 2026 Marimo. All rights reserved.
+"""Classify how a kernel subprocess exited.
+
+Used to turn a raw ``multiprocessing.Process.exitcode`` into a structured
+``KernelExitInfo`` with a human-readable message. Best-effort cgroup
+inspection adds memory diagnostics on Linux when the kernel was SIGKILLed,
+which on a container is almost always the cgroup OOM killer.
+"""
+
+from __future__ import annotations
+
+import os
+import signal
+import sys
+
+from marimo._session.types import KernelExitInfo
+
+# POSIX signal numbers used for kernel-exit classification. We pull them from
+# the ``signal`` module so the branches read naturally, with numeric fallbacks
+# for Windows (which doesn't define SIGKILL). The surrounding code is gated to
+# Linux at runtime, so the fallbacks only matter for tests that monkeypatch
+# ``sys.platform``.
+_SIGKILL = getattr(signal, "SIGKILL", 9)
+_SIGSEGV = getattr(signal, "SIGSEGV", 11)
+
+# Set by the container runtime / launcher to the pod's memory limit in
+# mebibytes (binary, 1 MiB = 1024 * 1024 bytes). Used purely to enrich OOM
+# messages; safe to leave unset.
+_MEMORY_LIMIT_ENV = "MARIMO_KERNEL_MEMORY_LIMIT_MIB"
+
+# cgroup v1 paths (used in molab's gVisor sandbox today). v2 uses different
+# filenames under /sys/fs/cgroup directly; we try both.
+_CGROUP_V1_FAILCNT = "/sys/fs/cgroup/memory/memory.failcnt"
+_CGROUP_V1_PEAK = "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
+_CGROUP_V2_EVENTS = "/sys/fs/cgroup/memory.events"
+_CGROUP_V2_PEAK = "/sys/fs/cgroup/memory.peak"
+
+
+_RESTART_HINT = "Click Restart to start a fresh kernel."
+
+
+def classify_kernel_exit(exitcode: int | None) -> KernelExitInfo:
+ """Return a KernelExitInfo describing how the kernel exited.
+
+ Messages are end-user-facing: each one names the cause in plain language
+ and ends with a concrete next step (typically pressing Restart, sometimes
+ paired with advice on freeing memory or trimming inputs).
+ """
+ if exitcode is None:
+ return KernelExitInfo(
+ exitcode=None,
+ cause="unknown",
+ message=f"The kernel stopped unexpectedly. {_RESTART_HINT}",
+ )
+ if exitcode == 0:
+ return KernelExitInfo(
+ exitcode=0,
+ cause="success",
+ message=f"The kernel finished cleanly. {_RESTART_HINT}",
+ )
+ if exitcode > 0:
+ return KernelExitInfo(
+ exitcode=exitcode,
+ cause="exit",
+ message=(
+ f"The kernel exited with error code {exitcode}. "
+ f"{_RESTART_HINT}"
+ ),
+ )
+
+ # ``exitcode < 0`` only occurs on POSIX (Windows never returns negative
+ # exitcodes). The signal-name and cgroup OOM logic below assumes Linux
+ # conventions (POSIX signal numbers per multiprocessing.Process.exitcode,
+ # see https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Process.exitcode,
+ # and Linux-specific cgroup files under /sys/fs/cgroup). On other
+ # platforms (including darwin) we have no way to confirm OOM, so we
+ # surface a generic failure rather than guessing -- claiming OOM for
+ # every SIGKILL would mislead users when the real cause is a crash,
+ # ``kill -9``, or something else.
+ if sys.platform != "linux":
+ return KernelExitInfo(
+ exitcode=exitcode,
+ cause="abnormal",
+ message=(f"The kernel was stopped unexpectedly. {_RESTART_HINT}"),
+ )
+
+ signal_num = -exitcode
+ if signal_num == _SIGKILL:
+ peak_mib = _peak_rss_mib()
+ if _was_oom_killed():
+ limit_mib = _memory_limit_mib()
+ usage = (
+ f" Notebook used {peak_mib} MiB of the {limit_mib} MiB limit."
+ if peak_mib is not None and limit_mib is not None
+ else f" Peak memory: {peak_mib} MiB."
+ if peak_mib is not None
+ else ""
+ )
+ return KernelExitInfo(
+ exitcode=exitcode,
+ cause="oom",
+ message=(
+ "The kernel ran out of memory and was stopped."
+ + usage
+ + " Free large variables (e.g. `del x; import gc; "
+ "gc.collect()`), reduce data sizes, or request a larger "
+ f"sandbox. {_RESTART_HINT}"
+ ),
+ )
+ # SIGKILL without cgroup-confirmed OOM: the cause is genuinely
+ # unknown (could be ``kill -9``, an external OOM-killer outside our
+ # cgroup, a crash, etc.). Be honest rather than guessing OOM.
+ return KernelExitInfo(
+ exitcode=exitcode,
+ cause="sigkill",
+ message=(f"The kernel was stopped unexpectedly. {_RESTART_HINT}"),
+ )
+ elif signal_num == _SIGSEGV:
+ return KernelExitInfo(
+ exitcode=exitcode,
+ cause="segfault",
+ message=(
+ "A native extension crashed the kernel (segmentation "
+ f"fault). {_RESTART_HINT} If it recurs, try a different "
+ "version of the package that triggered it."
+ ),
+ )
+ return KernelExitInfo(
+ exitcode=exitcode,
+ cause=f"signal_{signal_num}",
+ message=(
+ f"The kernel was stopped by the operating system (signal "
+ f"{signal_num}). {_RESTART_HINT}"
+ ),
+ )
+
+
+def _was_oom_killed() -> bool:
+ failcnt = _read_int(_CGROUP_V1_FAILCNT)
+ if failcnt is not None:
+ return failcnt > 0
+ oom_kill = _read_kv(_CGROUP_V2_EVENTS, "oom_kill")
+ if oom_kill is not None:
+ return oom_kill > 0
+ return False
+
+
+def _peak_rss_mib() -> int | None:
+ for path in (_CGROUP_V1_PEAK, _CGROUP_V2_PEAK):
+ value = _read_int(path)
+ if value is not None:
+ return value // (1024 * 1024)
+ return None
+
+
+def _memory_limit_mib() -> int | None:
+ raw = os.environ.get(_MEMORY_LIMIT_ENV)
+ if raw is None:
+ return None
+ try:
+ return int(raw)
+ except ValueError:
+ return None
+
+
+def _read_int(path: str) -> int | None:
+ try:
+ with open(path, encoding="utf-8") as f:
+ return int(f.read().strip())
+ except (OSError, ValueError):
+ return None
+
+
+def _read_kv(path: str, key: str) -> int | None:
+ try:
+ with open(path, encoding="utf-8") as f:
+ for line in f:
+ parts = line.split()
+ if len(parts) == 2 and parts[0] == key:
+ try:
+ return int(parts[1])
+ except ValueError:
+ return None
+ except OSError:
+ return None
+ return None
diff --git a/marimo/_session/managers/ipc.py b/marimo/_session/managers/ipc.py
index 69c8d0d664..9510401078 100644
--- a/marimo/_session/managers/ipc.py
+++ b/marimo/_session/managers/ipc.py
@@ -427,6 +427,11 @@ class _SubprocessWrapper(ProcessLike):
def pid(self) -> int | None:
return self._process.pid
+ @property
+ def exitcode(self) -> int | None:
+ """Mirror multiprocessing.Process.exitcode for exit diagnostics."""
+ return self._process.poll()
+
def is_alive(self) -> bool:
return self._process.poll() is None
diff --git a/marimo/_session/session.py b/marimo/_session/session.py
index 6e4a88992d..eedc216595 100644
--- a/marimo/_session/session.py
+++ b/marimo/_session/session.py
@@ -45,6 +45,7 @@ from marimo._session.extensions.types import (
ExtensionRegistry,
SessionExtension,
)
+from marimo._session.kernel_exit import classify_kernel_exit
from marimo._session.managers import (
KernelManagerImpl,
QueueManagerImpl,
@@ -54,6 +55,7 @@ from marimo._session.notebook import AppFileManager
from marimo._session.room import Room
from marimo._session.state.session_view import SessionView
from marimo._session.types import (
+ KernelExitInfo,
KernelManager,
KernelState,
QueueManager,
@@ -338,6 +340,16 @@ class SessionImpl(Session):
"""Get the PID of the kernel."""
return self._kernel_manager.pid
+ def kernel_exit_info(self) -> KernelExitInfo | None:
+ """Describe how the kernel exited."""
+ task = self._kernel_manager.kernel_task
+ if task is None or task.is_alive():
+ return None
+ # ``exitcode`` is provided by multiprocessing.Process; threads don't
+ # have one, so we treat absence as "unknown".
+ exitcode = getattr(task, "exitcode", None)
+ return classify_kernel_exit(exitcode)
+
def put_control_request(
self,
request: commands.CommandMessage,
diff --git a/marimo/_session/types.py b/marimo/_session/types.py
index 6df5b36490..641544730a 100644
--- a/marimo/_session/types.py
+++ b/marimo/_session/types.py
@@ -9,6 +9,7 @@ implementations.
from __future__ import annotations
import contextlib
+from dataclasses import dataclass
from enum import Enum
from typing import TYPE_CHECKING, Any, Protocol
@@ -104,6 +105,25 @@ class KernelState(Enum):
STOPPED = "stopped"
+@dataclass(frozen=True)
+class KernelExitInfo:
+ """Information about how a kernel exited.
+
+ Populated after the kernel task has stopped. ``exitcode`` follows the
+ convention of ``multiprocessing.Process.exitcode``: ``>= 0`` for a normal
+ exit with that status, and ``< 0`` if the process was terminated by signal
+ ``-exitcode``. ``None`` means the exit status is unavailable -- either the
+ task has not yet terminated, or the underlying task type does not expose
+ one (e.g. threads). ``cause`` is a short machine-readable tag and
+ ``message`` is a human-readable one-liner suitable for logs or end-user
+ display.
+ """
+
+ exitcode: int | None
+ cause: str
+ message: str
+
+
class Session(Protocol):
"""Protocol for session management."""
@@ -132,6 +152,13 @@ class Session(Protocol):
"""Get the PID of the kernel."""
...
+ def kernel_exit_info(self) -> KernelExitInfo | None:
+ """Describe how the kernel exited, or ``None`` if it is still running.
+
+ Only meaningful once ``kernel_state() == KernelState.STOPPED``.
+ """
+ ...
+
def try_interrupt(self) -> None:
"""Try to interrupt the kernel."""
...
diff --git a/tests/_session/extensions/test_extensions.py b/tests/_session/extensions/test_extensions.py
index 3d0cc1a5d3..451a2f7168 100644
--- a/tests/_session/extensions/test_extensions.py
+++ b/tests/_session/extensions/test_extensions.py
@@ -8,6 +8,7 @@ from unittest.mock import Mock, patch
import pytest
+from marimo._messaging.notification import BannerNotification
from marimo._session.events import SessionEventBus
from marimo._session.extensions.extensions import (
CacheMode,
@@ -24,7 +25,7 @@ from marimo._session.extensions.types import (
ExtensionRegistry,
)
from marimo._session.model import ConnectionState, SessionMode
-from marimo._session.types import KernelState
+from marimo._session.types import KernelExitInfo, KernelState
from marimo._types.ids import CellId_t, RequestId, SessionId
@@ -74,12 +75,30 @@ class TestHeartbeatExtension:
async def test_detects_dead_kernel(self, mock_session, event_bus) -> None:
"""Test that heartbeat detects when kernel dies and closes session."""
mock_session.kernel_state.return_value = KernelState.STOPPED
+ mock_session.kernel_exit_info.return_value = KernelExitInfo(
+ exitcode=-9,
+ cause="oom",
+ message=(
+ "The kernel ran out of memory and was stopped. "
+ "Notebook used 1952 MiB of the 2048 MiB limit. "
+ "Click Restart to start a fresh kernel."
+ ),
+ )
extension = HeartbeatExtension()
extension.on_attach(mock_session, event_bus)
await asyncio.sleep(1.5)
mock_session.close.assert_called_once()
+ # A persistent banner is broadcast before the session closes, so the
+ # frontend can show the cause instead of just a "disconnected" toast.
+ mock_session.notify.assert_called_once()
+ banner = mock_session.notify.call_args.args[0]
+ assert isinstance(banner, BannerNotification)
+ assert banner.variant == "danger"
+ assert banner.action == "restart"
+ assert "out of memory" in banner.description
+ assert "restart" in banner.description.lower()
extension.on_detach()
diff --git a/tests/_session/managers/test_ipc.py b/tests/_session/managers/test_ipc.py
index 734a05fe6a..07ef5672fe 100644
--- a/tests/_session/managers/test_ipc.py
+++ b/tests/_session/managers/test_ipc.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import os
import time
from pathlib import Path
+from unittest.mock import MagicMock
import pytest
@@ -107,6 +108,17 @@ class TestIPCKernelManagerImpl:
assert kernel_manager.venv_python == "/path/to/sandbox/venv/python"
+class TestSubprocessWrapper:
+ def test_exitcode_uses_popen_returncode(self) -> None:
+ from marimo._session.managers.ipc import _SubprocessWrapper
+
+ process = MagicMock()
+ process.poll.return_value = -9
+ wrapper = _SubprocessWrapper(process)
+
+ assert wrapper.exitcode == -9
+
+
@pytest.mark.requires("zmq")
class TestIPCQueueManagerImpl:
def test_from_ipc_factory(self) -> None:
diff --git a/tests/_session/test_kernel_exit.py b/tests/_session/test_kernel_exit.py
new file mode 100644
index 0000000000..568ad0b5aa
--- /dev/null
+++ b/tests/_session/test_kernel_exit.py
@@ -0,0 +1,135 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import sys
+from typing import TYPE_CHECKING
+
+from marimo._session import kernel_exit
+from marimo._session.kernel_exit import classify_kernel_exit
+
+if TYPE_CHECKING:
+ import pytest
+
+
+def _force_linux(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Pin sys.platform so signal/cgroup classification is exercised on any host."""
+ monkeypatch.setattr(sys, "platform", "linux")
+
+
+def test_unknown_when_exitcode_none() -> None:
+ info = classify_kernel_exit(None)
+ assert info.cause == "unknown"
+ assert info.exitcode is None
+
+
+def test_success_exit() -> None:
+ info = classify_kernel_exit(0)
+ assert info.cause == "success"
+ assert info.exitcode == 0
+
+
+def test_nonzero_exit() -> None:
+ info = classify_kernel_exit(2)
+ assert info.cause == "exit"
+ assert "code 2" in info.message
+
+
+def test_segfault_on_linux(monkeypatch: pytest.MonkeyPatch) -> None:
+ _force_linux(monkeypatch)
+ info = classify_kernel_exit(-11)
+ assert info.cause == "segfault"
+ assert "segmentation" in info.message.lower()
+
+
+def test_sigkill_without_oom_signal(monkeypatch: pytest.MonkeyPatch) -> None:
+ # No cgroup files readable -> SIGKILL with no OOM evidence.
+ # We must NOT claim OOM since the real cause could be a manual kill,
+ # an external OOM-killer outside our cgroup, a crash, etc.
+ _force_linux(monkeypatch)
+ monkeypatch.setattr(kernel_exit, "_read_int", lambda _path: None)
+ monkeypatch.setattr(kernel_exit, "_read_kv", lambda _path, _key: None)
+ info = classify_kernel_exit(-9)
+ assert info.cause == "sigkill"
+ assert "out of memory" not in info.message.lower()
+ assert "restart" in info.message.lower()
+
+
+def test_sigkill_with_cgroup_oom_v1(monkeypatch: pytest.MonkeyPatch) -> None:
+ _force_linux(monkeypatch)
+
+ def fake_read_int(path: str) -> int | None:
+ if "failcnt" in path:
+ return 5
+ if "max_usage" in path:
+ return 1952 * 1024 * 1024
+ return None
+
+ monkeypatch.setattr(kernel_exit, "_read_int", fake_read_int)
+ monkeypatch.setattr(kernel_exit, "_read_kv", lambda _path, _key: None)
+ monkeypatch.setenv(kernel_exit._MEMORY_LIMIT_ENV, "2048")
+
+ info = classify_kernel_exit(-9)
+ assert info.cause == "oom"
+ assert "1952 MiB" in info.message
+ assert "2048 MiB" in info.message
+
+
+def test_sigkill_with_cgroup_oom_v2(monkeypatch: pytest.MonkeyPatch) -> None:
+ _force_linux(monkeypatch)
+
+ def fake_read_int(path: str) -> int | None:
+ if path == kernel_exit._CGROUP_V2_PEAK:
+ return 100 * 1024 * 1024
+ return None
+
+ def fake_read_kv(path: str, key: str) -> int | None:
+ if path == kernel_exit._CGROUP_V2_EVENTS and key == "oom_kill":
+ return 1
+ return None
+
+ monkeypatch.setattr(kernel_exit, "_read_int", fake_read_int)
+ monkeypatch.setattr(kernel_exit, "_read_kv", fake_read_kv)
+ monkeypatch.delenv(kernel_exit._MEMORY_LIMIT_ENV, raising=False)
+
+ info = classify_kernel_exit(-9)
+ assert info.cause == "oom"
+ assert "100 MiB" in info.message
+
+
+def test_other_signal_on_linux(monkeypatch: pytest.MonkeyPatch) -> None:
+ _force_linux(monkeypatch)
+ info = classify_kernel_exit(-15) # SIGTERM
+ assert info.cause == "signal_15"
+ assert "signal 15" in info.message
+ assert "restart" in info.message.lower()
+
+
+def test_negative_exitcode_on_non_linux(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # On Windows/macOS we have no way to confirm OOM; surface a generic
+ # failure rather than guessing. Don't leak raw signal jargon either.
+ monkeypatch.setattr(sys, "platform", "win32")
+ info = classify_kernel_exit(-9)
+ assert info.cause == "abnormal"
+ assert "out of memory" not in info.message.lower()
+ assert "restart" in info.message.lower()
+ assert "SIGKILL" not in info.message
+ assert "exitcode" not in info.message.lower()
+
+ monkeypatch.setattr(sys, "platform", "darwin")
+ info = classify_kernel_exit(-11)
+ assert info.cause == "abnormal"
+ assert "out of memory" not in info.message.lower()
+ assert "restart" in info.message.lower()
+
+
+def test_positive_exitcode_on_non_linux(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Positive exitcodes still classify the same way on every platform.
+ monkeypatch.setattr(sys, "platform", "win32")
+ info = classify_kernel_exit(1)
+ assert info.cause == "exit"
+ assert "code 1" in info.message
+ assert "restart" in info.message.lower()