Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: finalize cells after post-execution hook interrupts (#10792)

Myles Scolnick104e5dcd38e5
diff --git a/marimo/_runtime/runner/cell_runner.py b/marimo/_runtime/runner/cell_runner.py
index 45295ae860..53b73f48b2 100644
--- a/marimo/_runtime/runner/cell_runner.py
+++ b/marimo/_runtime/runner/cell_runner.py
@@ -62,6 +62,7 @@ if TYPE_CHECKING:
from collections import deque
from marimo._ast.cell import CellImpl
+ from marimo._runtime.runner.hook_context import PostExecutionHookContext
from marimo._runtime.runner.hooks import NotebookCellHooks
from marimo._runtime.state import State
@@ -754,6 +755,33 @@ class Runner:
return defining_cell_id
return None
+ def _run_post_execution_hooks(
+ self,
+ cell: CellImpl,
+ ctx: PostExecutionHookContext,
+ run_result: RunResult,
+ ) -> None:
+ try:
+ for post_hook in self._hooks.post_execution_hooks:
+ try:
+ post_hook(cell, ctx, run_result)
+ except KeyboardInterrupt:
+ self.interrupted = True
+ LOGGER.info(
+ "Cell %s interrupted during post-execution hook",
+ cell.cell_id,
+ )
+ finally:
+ # Cleanup must complete even if interrupted after updating local
+ # state but before broadcasting it to the frontend.
+ while True:
+ try:
+ for finalize in self._hooks.finalization_hooks:
+ finalize(cell, ctx, run_result)
+ break
+ except KeyboardInterrupt:
+ self.interrupted = True
+
async def _run_one(
self,
cell_id: CellId_t,
@@ -770,8 +798,9 @@ class Runner:
with self.execution_context(cell_id) as exc_ctx:
run_result = await self.run(cell_id)
run_result.accumulated_output = exc_ctx.output
- for post_hook in self._hooks.post_execution_hooks:
- post_hook(cell, post_exec_ctx, run_result)
+ self._run_post_execution_hooks(
+ cell, post_exec_ctx, run_result
+ )
except KeyboardInterrupt:
LOGGER.error(
"A keyboard interrupt was raised but not handled by "
@@ -779,8 +808,7 @@ class Runner:
)
else:
run_result = await self.run(cell_id)
- for post_hook in self._hooks.post_execution_hooks:
- post_hook(cell, post_exec_ctx, run_result)
+ self._run_post_execution_hooks(cell, post_exec_ctx, run_result)
async def run_all(self) -> None:
from marimo._runtime.runner.hook_context import (
diff --git a/marimo/_runtime/runner/hooks.py b/marimo/_runtime/runner/hooks.py
index c6441b0d0c..ef5bf35b86 100644
--- a/marimo/_runtime/runner/hooks.py
+++ b/marimo/_runtime/runner/hooks.py
@@ -44,6 +44,7 @@ class HookPhase(str, Enum):
PREPARATION = "preparation"
PRE_EXECUTION = "pre_execution"
POST_EXECUTION = "post_execution"
+ FINALIZATION = "finalization"
ON_FINISH = "on_finish"
@@ -96,7 +97,8 @@ class NotebookCellHooks:
1. preparation_hooks: Run once before the runner starts
2. pre_execution_hooks: Run before each cell executes
3. post_execution_hooks: Run after each cell executes
- 4. on_finish_hooks: Run once after all cells complete
+ 4. finalization_hooks: Always run after post-execution, retrying interrupts
+ 5. on_finish_hooks: Run once after all cells complete
"""
def __init__(
@@ -132,6 +134,10 @@ class NotebookCellHooks:
) -> None:
self._add(HookPhase.POST_EXECUTION, hook, priority)
+ def add_finalization(self, hook: PostExecutionHook) -> None:
+ """Register cleanup that is safe to repeat after an interrupt."""
+ self._add(HookPhase.FINALIZATION, hook, Priority.FINAL)
+
def add_on_finish(
self, hook: OnFinishHook, priority: Priority = Priority.NORMAL
) -> None:
@@ -155,6 +161,12 @@ class NotebookCellHooks:
"Sequence[PostExecutionHook]", self._get(HookPhase.POST_EXECUTION)
)
+ @property
+ def finalization_hooks(self) -> Sequence[PostExecutionHook]:
+ return cast(
+ "Sequence[PostExecutionHook]", self._get(HookPhase.FINALIZATION)
+ )
+
@property
def on_finish_hooks(self) -> Sequence[OnFinishHook]:
return cast("Sequence[OnFinishHook]", self._get(HookPhase.ON_FINISH))
@@ -174,6 +186,7 @@ def create_default_hooks() -> NotebookCellHooks:
"""
from marimo._runtime.runner.hooks_on_finish import ON_FINISH_HOOKS
from marimo._runtime.runner.hooks_post_execution import (
+ FINALIZATION_HOOKS,
POST_EXECUTION_HOOKS,
)
from marimo._runtime.runner.hooks_pre_execution import (
@@ -193,4 +206,6 @@ def create_default_hooks() -> NotebookCellHooks:
for phase, hook_list in defaults:
for hook in hook_list:
hooks._add(phase, hook, Priority.NORMAL)
+ for hook in FINALIZATION_HOOKS:
+ hooks.add_finalization(hook)
return hooks
diff --git a/marimo/_runtime/runner/hooks_post_execution.py b/marimo/_runtime/runner/hooks_post_execution.py
index b3be4ceed0..eb5904497a 100644
--- a/marimo/_runtime/runner/hooks_post_execution.py
+++ b/marimo/_runtime/runner/hooks_post_execution.py
@@ -560,6 +560,11 @@ POST_EXECUTION_HOOKS: list[PostExecutionHook] = [
_broadcast_outputs,
_reset_matplotlib_context,
_delete_local_variables,
+]
+
+
+# These hooks may be retried after an interrupt, so they must be safe to repeat.
+FINALIZATION_HOOKS: list[PostExecutionHook] = [
# Flush buffered console output so that stderr/stdout arrives at the
# frontend before the cell transitions to idle.
_flush_console,
@@ -567,6 +572,4 @@ POST_EXECUTION_HOOKS: list[PostExecutionHook] = [
# other hooks take a long time (broadcast outputs can take a long time
# if a formatter is slow).
_set_status_idle,
- # NB. Other hooks are added ad-hoc or manually due to priority.
- # Consider implementing priority sort to keep everything more centralized.
]
diff --git a/tests/_runtime/runner/test_cell_runner.py b/tests/_runtime/runner/test_cell_runner.py
index 9973b8a63c..c7315c7808 100644
--- a/tests/_runtime/runner/test_cell_runner.py
+++ b/tests/_runtime/runner/test_cell_runner.py
@@ -4,13 +4,21 @@ from typing import Any
import pytest
+from marimo._ast.cell import CellImpl
from marimo._dependencies.dependencies import DependencyManager
from marimo._messaging.errors import MarimoSQLError
+from marimo._messaging.notification_utils import CellNotificationUtils
from marimo._runtime.capture import capture_stderr
from marimo._runtime.runner.cell_runner import Runner
-from marimo._runtime.runner.hooks import NotebookCellHooks
+from marimo._runtime.runner.hook_context import PostExecutionHookContext
+from marimo._runtime.runner.hooks import (
+ NotebookCellHooks,
+ Priority,
+ create_default_hooks,
+)
+from marimo._runtime.runner.result import RunResult
from marimo._runtime.runtime import Kernel
-from tests.conftest import ExecReqProvider
+from tests.conftest import ExecReqProvider, MockedKernel
async def test_cell_output(
@@ -489,6 +497,173 @@ async def test_runner_interrupted_flag_flips_on_async_cell_cancellation(
assert runner.interrupted is True
[email protected]("with_execution_context", [True, False])
[email protected]("interrupt_count", [1, 2])
+async def test_post_execution_interrupt_still_finalizes_cell(
+ mocked_kernel: MockedKernel,
+ exec_req: ExecReqProvider,
+ monkeypatch: pytest.MonkeyPatch,
+ with_execution_context: bool,
+ interrupt_count: int,
+) -> None:
+ k = mocked_kernel.k
+ await k.run(
+ [
+ er := exec_req.get("x = 123; x"),
+ child := exec_req.get("y = x + 1"),
+ ]
+ )
+ del k.globals["y"]
+ mocked_kernel.stream.messages.clear()
+ cell = k.graph.cells[er.cell_id]
+ events: list[str] = []
+
+ def interrupt(
+ cell: CellImpl, ctx: PostExecutionHookContext, result: RunResult
+ ) -> None:
+ del ctx
+ if cell.cell_id != er.cell_id:
+ return
+ assert cell.runtime_state == "running"
+ assert (result.output, result.exception) == (123, None)
+ events.append("interrupt")
+ raise KeyboardInterrupt
+
+ def flush_console() -> None:
+ assert cell.runtime_state == "running"
+ events.append("flush")
+
+ monkeypatch.setattr(k.stream, "flush_console", flush_console)
+ hooks = create_default_hooks()
+ for _ in range(interrupt_count):
+ hooks.add_post_execution(interrupt, Priority.EARLY)
+ runner = Runner(
+ roots={er.cell_id},
+ graph=k.graph,
+ glbls=k.globals,
+ debugger=k.debugger,
+ hooks=hooks,
+ execution_context=(
+ k._install_execution_context if with_execution_context else None
+ ),
+ )
+
+ await runner.run_all()
+
+ assert (cell.runtime_state, cell.run_result_status) == ("idle", "success")
+ assert events == ["interrupt"] * interrupt_count + ["flush"]
+ assert [
+ op.status
+ for op in mocked_kernel.stream.cell_notifications
+ if op.cell_id == er.cell_id and op.status is not None
+ ] == ["queued", "running", "idle"]
+ assert runner.interrupted
+ assert "y" not in k.globals
+ assert k.graph.cells[child.cell_id].runtime_state == "idle"
+
+
[email protected]("with_execution_context", [True, False])
[email protected]("interrupted_cleanup", ["flush", "idle"])
+async def test_interrupt_during_finalization_still_broadcasts_idle(
+ mocked_kernel: MockedKernel,
+ exec_req: ExecReqProvider,
+ monkeypatch: pytest.MonkeyPatch,
+ with_execution_context: bool,
+ interrupted_cleanup: str,
+) -> None:
+ k = mocked_kernel.k
+ await k.run([er := exec_req.get("123")])
+ mocked_kernel.stream.messages.clear()
+ cell = k.graph.cells[er.cell_id]
+ interruptions = 0
+ events: list[str] = []
+ broadcast_status = CellNotificationUtils.broadcast_status
+
+ def maybe_interrupt(cleanup: str) -> None:
+ nonlocal interruptions
+ if cleanup == interrupted_cleanup and interruptions < 2:
+ interruptions += 1
+ raise KeyboardInterrupt
+
+ def flush_console() -> None:
+ maybe_interrupt("flush")
+ events.append("flush")
+
+ def broadcast_status_with_interrupt(*args: Any, **kwargs: Any) -> None:
+ if kwargs.get("status") == "idle":
+ # The local state is already idle; the frontend still needs
+ # the notification even when that state hasn't changed.
+ assert cell.runtime_state == "idle"
+ maybe_interrupt("idle")
+ events.append("idle")
+ broadcast_status(*args, **kwargs)
+
+ monkeypatch.setattr(k.stream, "flush_console", flush_console)
+ monkeypatch.setattr(
+ CellNotificationUtils,
+ "broadcast_status",
+ broadcast_status_with_interrupt,
+ )
+ runner = Runner(
+ roots={er.cell_id},
+ graph=k.graph,
+ glbls=k.globals,
+ debugger=k.debugger,
+ hooks=create_default_hooks(),
+ execution_context=(
+ k._install_execution_context if with_execution_context else None
+ ),
+ )
+
+ await runner.run_all()
+
+ assert interruptions == 2
+ assert events[-2:] == ["flush", "idle"]
+ assert (cell.runtime_state, cell.run_result_status) == ("idle", "success")
+ assert [
+ op.status
+ for op in mocked_kernel.stream.cell_notifications
+ if op.cell_id == er.cell_id and op.status is not None
+ ] == ["queued", "running", "idle"]
+ assert runner.interrupted
+
+
+async def test_post_execution_error_still_finalizes_cell(
+ mocked_kernel: MockedKernel,
+ exec_req: ExecReqProvider,
+) -> None:
+ k = mocked_kernel.k
+ await k.run([er := exec_req.get("123")])
+ mocked_kernel.stream.messages.clear()
+
+ def fail(
+ cell: CellImpl, ctx: PostExecutionHookContext, result: RunResult
+ ) -> None:
+ del cell, ctx, result
+ raise ValueError("post-execution failed")
+
+ hooks = create_default_hooks()
+ hooks.add_post_execution(fail)
+ runner = Runner(
+ roots={er.cell_id},
+ graph=k.graph,
+ glbls=k.globals,
+ debugger=k.debugger,
+ hooks=hooks,
+ execution_context=k._install_execution_context,
+ )
+
+ with pytest.raises(ValueError, match="post-execution failed"):
+ await runner.run_all()
+
+ assert k.graph.cells[er.cell_id].runtime_state == "idle"
+ assert [
+ op.status
+ for op in mocked_kernel.stream.cell_notifications
+ if op.cell_id == er.cell_id and op.status is not None
+ ] == ["queued", "running", "idle"]
+
+
async def test_run_all_swallows_sigint_raise_and_fires_on_finish(
execution_kernel: Kernel,
exec_req: ExecReqProvider,
diff --git a/tests/_runtime/runner/test_hooks.py b/tests/_runtime/runner/test_hooks.py
index 8915421159..65ffe05d3d 100644
--- a/tests/_runtime/runner/test_hooks.py
+++ b/tests/_runtime/runner/test_hooks.py
@@ -58,20 +58,24 @@ class TestCreateDefaultHooks:
assert len(hooks.preparation_hooks) > 0
assert len(hooks.pre_execution_hooks) > 0
assert len(hooks.post_execution_hooks) > 0
+ assert len(hooks.finalization_hooks) > 0
assert len(hooks.on_finish_hooks) > 0
- def test_set_status_idle_is_last_post_execution_hook(self) -> None:
- """Verify _set_status_idle is the last hook in POST_EXECUTION_HOOKS.
+ def test_set_status_idle_is_last_finalization_hook(self) -> None:
+ """Verify _set_status_idle runs after flushing console output.
This is important because status should only be set to idle after all
other post-execution work (like broadcasting outputs) is complete.
"""
from marimo._runtime.runner.hooks_post_execution import (
- POST_EXECUTION_HOOKS,
+ _flush_console,
_set_status_idle,
)
- assert POST_EXECUTION_HOOKS[-1] is _set_status_idle
+ hooks = create_default_hooks()
+ assert hooks.finalization_hooks == [_flush_console, _set_status_idle]
+ assert _flush_console not in hooks.post_execution_hooks
+ assert _set_status_idle not in hooks.post_execution_hooks
class TestSetRunResultStatus: