A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: finalize cells after post-execution hook interrupts (#10792)
Myles Scolnick104e5dcd38e5
diff --git a/marimo/_runtime/runner/cell_runner.py b/marimo/_runtime/runner/cell_runner.pyindex 45295ae860..53b73f48b2 100644--- a/marimo/_runtime/runner/cell_runner.py+++ b/marimo/_runtime/runner/cell_runner.py@@ -62,6 +62,7 @@ if TYPE_CHECKING:from collections import dequefrom marimo._ast.cell import CellImpl+ from marimo._runtime.runner.hook_context import PostExecutionHookContextfrom marimo._runtime.runner.hooks import NotebookCellHooksfrom marimo._runtime.state import State@@ -754,6 +755,33 @@ class Runner:return defining_cell_idreturn None+ def _run_post_execution_hooks(+ self,+ cell: CellImpl,+ ctx: PostExecutionHookContext,+ run_result: RunResult,+ ) -> None:+ try:+ for post_hook in self._hooks.post_execution_hooks:+ try:+ post_hook(cell, ctx, run_result)+ except KeyboardInterrupt:+ self.interrupted = True+ LOGGER.info(+ "Cell %s interrupted during post-execution hook",+ cell.cell_id,+ )+ finally:+ # Cleanup must complete even if interrupted after updating local+ # state but before broadcasting it to the frontend.+ while True:+ try:+ for finalize in self._hooks.finalization_hooks:+ finalize(cell, ctx, run_result)+ break+ except KeyboardInterrupt:+ self.interrupted = True+async def _run_one(self,cell_id: CellId_t,@@ -770,8 +798,9 @@ class Runner:with self.execution_context(cell_id) as exc_ctx:run_result = await self.run(cell_id)run_result.accumulated_output = exc_ctx.output- for post_hook in self._hooks.post_execution_hooks:- post_hook(cell, post_exec_ctx, run_result)+ self._run_post_execution_hooks(+ cell, post_exec_ctx, run_result+ )except KeyboardInterrupt:LOGGER.error("A keyboard interrupt was raised but not handled by "@@ -779,8 +808,7 @@ class Runner:)else:run_result = await self.run(cell_id)- for post_hook in self._hooks.post_execution_hooks:- post_hook(cell, post_exec_ctx, run_result)+ self._run_post_execution_hooks(cell, post_exec_ctx, run_result)async def run_all(self) -> None:from marimo._runtime.runner.hook_context import (diff --git a/marimo/_runtime/runner/hooks.py b/marimo/_runtime/runner/hooks.pyindex c6441b0d0c..ef5bf35b86 100644--- a/marimo/_runtime/runner/hooks.py+++ b/marimo/_runtime/runner/hooks.py@@ -44,6 +44,7 @@ class HookPhase(str, Enum):PREPARATION = "preparation"PRE_EXECUTION = "pre_execution"POST_EXECUTION = "post_execution"+ FINALIZATION = "finalization"ON_FINISH = "on_finish"@@ -96,7 +97,8 @@ class NotebookCellHooks:1. preparation_hooks: Run once before the runner starts2. pre_execution_hooks: Run before each cell executes3. post_execution_hooks: Run after each cell executes- 4. on_finish_hooks: Run once after all cells complete+ 4. finalization_hooks: Always run after post-execution, retrying interrupts+ 5. on_finish_hooks: Run once after all cells complete"""def __init__(@@ -132,6 +134,10 @@ class NotebookCellHooks:) -> None:self._add(HookPhase.POST_EXECUTION, hook, priority)+ def add_finalization(self, hook: PostExecutionHook) -> None:+ """Register cleanup that is safe to repeat after an interrupt."""+ self._add(HookPhase.FINALIZATION, hook, Priority.FINAL)+def add_on_finish(self, hook: OnFinishHook, priority: Priority = Priority.NORMAL) -> None:@@ -155,6 +161,12 @@ class NotebookCellHooks:"Sequence[PostExecutionHook]", self._get(HookPhase.POST_EXECUTION))+ @property+ def finalization_hooks(self) -> Sequence[PostExecutionHook]:+ return cast(+ "Sequence[PostExecutionHook]", self._get(HookPhase.FINALIZATION)+ )+@propertydef on_finish_hooks(self) -> Sequence[OnFinishHook]:return cast("Sequence[OnFinishHook]", self._get(HookPhase.ON_FINISH))@@ -174,6 +186,7 @@ def create_default_hooks() -> NotebookCellHooks:"""from marimo._runtime.runner.hooks_on_finish import ON_FINISH_HOOKSfrom marimo._runtime.runner.hooks_post_execution import (+ FINALIZATION_HOOKS,POST_EXECUTION_HOOKS,)from marimo._runtime.runner.hooks_pre_execution import (@@ -193,4 +206,6 @@ def create_default_hooks() -> NotebookCellHooks:for phase, hook_list in defaults:for hook in hook_list:hooks._add(phase, hook, Priority.NORMAL)+ for hook in FINALIZATION_HOOKS:+ hooks.add_finalization(hook)return hooksdiff --git a/marimo/_runtime/runner/hooks_post_execution.py b/marimo/_runtime/runner/hooks_post_execution.pyindex b3be4ceed0..eb5904497a 100644--- a/marimo/_runtime/runner/hooks_post_execution.py+++ b/marimo/_runtime/runner/hooks_post_execution.py@@ -560,6 +560,11 @@ POST_EXECUTION_HOOKS: list[PostExecutionHook] = [_broadcast_outputs,_reset_matplotlib_context,_delete_local_variables,+]+++# These hooks may be retried after an interrupt, so they must be safe to repeat.+FINALIZATION_HOOKS: list[PostExecutionHook] = [# Flush buffered console output so that stderr/stdout arrives at the# frontend before the cell transitions to idle._flush_console,@@ -567,6 +572,4 @@ POST_EXECUTION_HOOKS: list[PostExecutionHook] = [# other hooks take a long time (broadcast outputs can take a long time# if a formatter is slow)._set_status_idle,- # NB. Other hooks are added ad-hoc or manually due to priority.- # Consider implementing priority sort to keep everything more centralized.]diff --git a/tests/_runtime/runner/test_cell_runner.py b/tests/_runtime/runner/test_cell_runner.pyindex 9973b8a63c..c7315c7808 100644--- a/tests/_runtime/runner/test_cell_runner.py+++ b/tests/_runtime/runner/test_cell_runner.py@@ -4,13 +4,21 @@ from typing import Anyimport pytest+from marimo._ast.cell import CellImplfrom marimo._dependencies.dependencies import DependencyManagerfrom marimo._messaging.errors import MarimoSQLError+from marimo._messaging.notification_utils import CellNotificationUtilsfrom marimo._runtime.capture import capture_stderrfrom marimo._runtime.runner.cell_runner import Runner-from marimo._runtime.runner.hooks import NotebookCellHooks+from marimo._runtime.runner.hook_context import PostExecutionHookContext+from marimo._runtime.runner.hooks import (+ NotebookCellHooks,+ Priority,+ create_default_hooks,+)+from marimo._runtime.runner.result import RunResultfrom marimo._runtime.runtime import Kernel-from tests.conftest import ExecReqProvider+from tests.conftest import ExecReqProvider, MockedKernelasync def test_cell_output(@@ -489,6 +497,173 @@ async def test_runner_interrupted_flag_flips_on_async_cell_cancellation(assert runner.interrupted is True[email protected]("with_execution_context", [True, False])[email protected]("interrupt_count", [1, 2])+async def test_post_execution_interrupt_still_finalizes_cell(+ mocked_kernel: MockedKernel,+ exec_req: ExecReqProvider,+ monkeypatch: pytest.MonkeyPatch,+ with_execution_context: bool,+ interrupt_count: int,+) -> None:+ k = mocked_kernel.k+ await k.run(+ [+ er := exec_req.get("x = 123; x"),+ child := exec_req.get("y = x + 1"),+ ]+ )+ del k.globals["y"]+ mocked_kernel.stream.messages.clear()+ cell = k.graph.cells[er.cell_id]+ events: list[str] = []++ def interrupt(+ cell: CellImpl, ctx: PostExecutionHookContext, result: RunResult+ ) -> None:+ del ctx+ if cell.cell_id != er.cell_id:+ return+ assert cell.runtime_state == "running"+ assert (result.output, result.exception) == (123, None)+ events.append("interrupt")+ raise KeyboardInterrupt++ def flush_console() -> None:+ assert cell.runtime_state == "running"+ events.append("flush")++ monkeypatch.setattr(k.stream, "flush_console", flush_console)+ hooks = create_default_hooks()+ for _ in range(interrupt_count):+ hooks.add_post_execution(interrupt, Priority.EARLY)+ runner = Runner(+ roots={er.cell_id},+ graph=k.graph,+ glbls=k.globals,+ debugger=k.debugger,+ hooks=hooks,+ execution_context=(+ k._install_execution_context if with_execution_context else None+ ),+ )++ await runner.run_all()++ assert (cell.runtime_state, cell.run_result_status) == ("idle", "success")+ assert events == ["interrupt"] * interrupt_count + ["flush"]+ assert [+ op.status+ for op in mocked_kernel.stream.cell_notifications+ if op.cell_id == er.cell_id and op.status is not None+ ] == ["queued", "running", "idle"]+ assert runner.interrupted+ assert "y" not in k.globals+ assert k.graph.cells[child.cell_id].runtime_state == "idle"++[email protected]("with_execution_context", [True, False])[email protected]("interrupted_cleanup", ["flush", "idle"])+async def test_interrupt_during_finalization_still_broadcasts_idle(+ mocked_kernel: MockedKernel,+ exec_req: ExecReqProvider,+ monkeypatch: pytest.MonkeyPatch,+ with_execution_context: bool,+ interrupted_cleanup: str,+) -> None:+ k = mocked_kernel.k+ await k.run([er := exec_req.get("123")])+ mocked_kernel.stream.messages.clear()+ cell = k.graph.cells[er.cell_id]+ interruptions = 0+ events: list[str] = []+ broadcast_status = CellNotificationUtils.broadcast_status++ def maybe_interrupt(cleanup: str) -> None:+ nonlocal interruptions+ if cleanup == interrupted_cleanup and interruptions < 2:+ interruptions += 1+ raise KeyboardInterrupt++ def flush_console() -> None:+ maybe_interrupt("flush")+ events.append("flush")++ def broadcast_status_with_interrupt(*args: Any, **kwargs: Any) -> None:+ if kwargs.get("status") == "idle":+ # The local state is already idle; the frontend still needs+ # the notification even when that state hasn't changed.+ assert cell.runtime_state == "idle"+ maybe_interrupt("idle")+ events.append("idle")+ broadcast_status(*args, **kwargs)++ monkeypatch.setattr(k.stream, "flush_console", flush_console)+ monkeypatch.setattr(+ CellNotificationUtils,+ "broadcast_status",+ broadcast_status_with_interrupt,+ )+ runner = Runner(+ roots={er.cell_id},+ graph=k.graph,+ glbls=k.globals,+ debugger=k.debugger,+ hooks=create_default_hooks(),+ execution_context=(+ k._install_execution_context if with_execution_context else None+ ),+ )++ await runner.run_all()++ assert interruptions == 2+ assert events[-2:] == ["flush", "idle"]+ assert (cell.runtime_state, cell.run_result_status) == ("idle", "success")+ assert [+ op.status+ for op in mocked_kernel.stream.cell_notifications+ if op.cell_id == er.cell_id and op.status is not None+ ] == ["queued", "running", "idle"]+ assert runner.interrupted+++async def test_post_execution_error_still_finalizes_cell(+ mocked_kernel: MockedKernel,+ exec_req: ExecReqProvider,+) -> None:+ k = mocked_kernel.k+ await k.run([er := exec_req.get("123")])+ mocked_kernel.stream.messages.clear()++ def fail(+ cell: CellImpl, ctx: PostExecutionHookContext, result: RunResult+ ) -> None:+ del cell, ctx, result+ raise ValueError("post-execution failed")++ hooks = create_default_hooks()+ hooks.add_post_execution(fail)+ runner = Runner(+ roots={er.cell_id},+ graph=k.graph,+ glbls=k.globals,+ debugger=k.debugger,+ hooks=hooks,+ execution_context=k._install_execution_context,+ )++ with pytest.raises(ValueError, match="post-execution failed"):+ await runner.run_all()++ assert k.graph.cells[er.cell_id].runtime_state == "idle"+ assert [+ op.status+ for op in mocked_kernel.stream.cell_notifications+ if op.cell_id == er.cell_id and op.status is not None+ ] == ["queued", "running", "idle"]++async def test_run_all_swallows_sigint_raise_and_fires_on_finish(execution_kernel: Kernel,exec_req: ExecReqProvider,diff --git a/tests/_runtime/runner/test_hooks.py b/tests/_runtime/runner/test_hooks.pyindex 8915421159..65ffe05d3d 100644--- a/tests/_runtime/runner/test_hooks.py+++ b/tests/_runtime/runner/test_hooks.py@@ -58,20 +58,24 @@ class TestCreateDefaultHooks:assert len(hooks.preparation_hooks) > 0assert len(hooks.pre_execution_hooks) > 0assert len(hooks.post_execution_hooks) > 0+ assert len(hooks.finalization_hooks) > 0assert len(hooks.on_finish_hooks) > 0- def test_set_status_idle_is_last_post_execution_hook(self) -> None:- """Verify _set_status_idle is the last hook in POST_EXECUTION_HOOKS.+ def test_set_status_idle_is_last_finalization_hook(self) -> None:+ """Verify _set_status_idle runs after flushing console output.This is important because status should only be set to idle after allother post-execution work (like broadcasting outputs) is complete."""from marimo._runtime.runner.hooks_post_execution import (- POST_EXECUTION_HOOKS,+ _flush_console,_set_status_idle,)- assert POST_EXECUTION_HOOKS[-1] is _set_status_idle+ hooks = create_default_hooks()+ assert hooks.finalization_hooks == [_flush_console, _set_status_idle]+ assert _flush_console not in hooks.post_execution_hooks+ assert _set_status_idle not in hooks.post_execution_hooksclass TestSetRunResultStatus: