A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
Deliver a command's events after its backend call (#11205)
Trevor Manz9b14875ae48f
diff --git a/marimo/_core/notebook.py b/marimo/_core/notebook.pyindex f9b13c8177..c4e4d91a5e 100644--- a/marimo/_core/notebook.py+++ b/marimo/_core/notebook.py@@ -25,13 +25,17 @@ if TYPE_CHECKING:class Notebook:- """A notebook's state and commands.+ """A document, its results, and the runtime that produces them.- Snapshots preserve insertion order for runs and executions, and for- outputs within each execution.- Current cell results survive runtime stops and restarts. The host- chooses how long to retain completed ephemeral results.- Removing the notebook closes it; further commands raise `NotFound`.+ Clients attach to a notebook, edit it, and run it. Every change is+ published as an event so that each client sees the same notebook.+ Commands change it and call the backend, which reports back through+ `observe`.++ Subscribers hear about a command once it has reached the backend.+ Results survive runtime stops and restarts, and the host decides how+ long completed scratch results are kept. Commands on a notebook the+ host has removed raise `NotFound`."""def __init__(@@ -86,14 +90,15 @@ class Notebook:Returns the runtime and whether a new runtime was created.An existing runtime is returned unchanged, regardless of status."""- self._require_open()- if self._runtime is not None:- return self._runtime, False- runtime = self._new_runtime("starting", None, None)- self._lifecycle_request = request_id- self._set_runtime(runtime)- self._backend.start(runtime.id)- return runtime, True+ with self._stream.deferred():+ self._require_open()+ if self._runtime is not None:+ return self._runtime, False+ runtime = self._new_runtime("starting", None, None)+ self._lifecycle_request = request_id+ self._set_runtime(runtime)+ self._backend.start(runtime.id)+ return runtime, Truedef stop(self, *, request_id: str | None = None) -> protocol.Runtime:"""Stop the runtime for all attached clients, retaining its results.@@ -105,19 +110,20 @@ class Notebook:Raises:NoRuntime: If the notebook has no runtime."""- self._require_open()- runtime = self._require_runtime()- self._lifecycle_request = request_id- self._replacement = None- if runtime.status == "terminating":- return runtime- terminating = replace(runtime, status="terminating")- self._set_runtime(terminating)- if runtime.status == "failed":- self._end_runtime()- else:- self._backend.stop()- return terminating+ with self._stream.deferred():+ self._require_open()+ runtime = self._require_runtime()+ self._lifecycle_request = request_id+ self._replacement = None+ if runtime.status == "terminating":+ return runtime+ terminating = replace(runtime, status="terminating")+ self._set_runtime(terminating)+ if runtime.status == "failed":+ self._end_runtime()+ else:+ self._backend.stop()+ return terminatingdef restart(self, *, request_id: str | None = None) -> protocol.Runtime:"""Stop the runtime and start a replacement with a new ID.@@ -129,19 +135,20 @@ class Notebook:NoRuntime: If the notebook has no runtime.Conflict: If the runtime is not running."""- self._require_open()- runtime = self._require_runtime()- if runtime.status != "running":- raise Conflict(- f"the runtime is {runtime.status}; only a running one restarts"- )- self._lifecycle_request = request_id- replacement = self._new_runtime("starting", None, None)- self._replacement = replacement- terminating = replace(runtime, status="terminating")- self._set_runtime(terminating)- self._backend.stop()- return replacement+ with self._stream.deferred():+ self._require_open()+ runtime = self._require_runtime()+ if runtime.status != "running":+ raise Conflict(+ f"the runtime is {runtime.status}; only a running one restarts"+ )+ self._lifecycle_request = request_id+ replacement = self._new_runtime("starting", None, None)+ self._replacement = replacement+ terminating = replace(runtime, status="terminating")+ self._set_runtime(terminating)+ self._backend.stop()+ return replacementdef _require_runtime(self) -> protocol.Runtime:if self._runtime is None:@@ -175,24 +182,25 @@ class Notebook:NoRuntime: If the notebook has no runtime.Conflict: If the runtime is not running, or a run is in progress."""- self._require_open()- runtime = self._require_runtime()- if runtime.status != "running":- raise Conflict(f"the runtime is {runtime.status}.")- if (active := self._state.active_run) is not None:- raise Conflict(f"run {active.id} is in progress.")- run = protocol.Run(- id=mint(protocol.RunId, "run"),- notebook_id=self.id,- caused_by=None,- status="queued",- outcome=None,- started_at=None,- completed_at=None,- )- self._state.add_run(run, request_id)- self._backend.run(run.id, code)- return run+ with self._stream.deferred():+ self._require_open()+ runtime = self._require_runtime()+ if runtime.status != "running":+ raise Conflict(f"the runtime is {runtime.status}.")+ if (active := self._state.active_run) is not None:+ raise Conflict(f"run {active.id} is in progress.")+ run = protocol.Run(+ id=mint(protocol.RunId, "run"),+ notebook_id=self.id,+ caused_by=None,+ status="queued",+ outcome=None,+ started_at=None,+ completed_at=None,+ )+ self._state.add_run(run, request_id)+ self._backend.run(run.id, code)+ return rundef interrupt(self, run_id: str) -> protocol.Run:"""Request interruption of a run in progress.@@ -279,11 +287,11 @@ class Notebook:if runtime.status == "terminating" and self._replacement is not None:replacement, self._replacement = self._replacement, Nonerequest_id = self._lifecycle_request- self._end_runtime()- self._lifecycle_request = request_id- self._set_runtime(replacement)- assert self._runtime is not None- self._backend.start(self._runtime.id)+ with self._stream.deferred():+ self._end_runtime()+ self._lifecycle_request = request_id+ self._set_runtime(replacement)+ self._backend.start(replacement.id)elif runtime.status == "terminating":self._end_runtime()else:@@ -317,7 +325,10 @@ class Notebook:# ---- stream --------------------------------------------------------def _snapshot(self) -> list[Event]:- """Return notebook objects as snapshot events, parents first."""+ """Return notebook objects as snapshot events, parents first.++ Runs, executions, and outputs keep their insertion order.+ """bodies: list[protocol.Runtime | protocol.Attachment] = []if self._runtime is not None:bodies.append(self._runtime)diff --git a/marimo/_core/stream.py b/marimo/_core/stream.pyindex cbb545b7c2..310a0d5ea7 100644--- a/marimo/_core/stream.py+++ b/marimo/_core/stream.py@@ -9,7 +9,8 @@ from __future__ import annotationsimport loggingfrom collections import deque-from collections.abc import Callable, Collection+from collections.abc import Callable, Collection, Iterator+from contextlib import contextmanagerfrom copy import deepcopyfrom dataclasses import dataclassfrom datetime import datetime, timezone@@ -206,6 +207,23 @@ class Stream:return unsubscribe+ @contextmanager+ def deferred(self) -> Iterator[None]:+ """Deliver events published in the block once it ends.++ Events still join the history at once, so cursors and replay are+ unaffected. Nested blocks defer to the outermost one.+ """+ if self._dispatching:+ yield+ return+ self._dispatching = True+ try:+ yield+ finally:+ self._dispatching = False+ self._drain()+def _append(self, event: Event) -> Event:self._last_id += 1stamped = Event(@@ -218,15 +236,18 @@ class Stream:self._history.append(stamped)self._pending.append((stamped, tuple(self._listeners)))if not self._dispatching:- self._dispatching = True- try:- while self._pending:- pending, listeners = self._pending.popleft()- for listener in listeners:- try:- listener(deepcopy(pending))- except Exception:- LOGGER.exception("Error delivering host event")- finally:- self._dispatching = False+ self._drain()return deepcopy(stamped)++ def _drain(self) -> None:+ self._dispatching = True+ try:+ while self._pending:+ pending, listeners = self._pending.popleft()+ for listener in listeners:+ try:+ listener(deepcopy(pending))+ except Exception:+ LOGGER.exception("Error delivering host event")+ finally:+ self._dispatching = Falsediff --git a/tests/_core/test_stream.py b/tests/_core/test_stream.pyindex 6cd53055cc..25d1c8936b 100644--- a/tests/_core/test_stream.py+++ b/tests/_core/test_stream.py@@ -5,7 +5,7 @@ from __future__ import annotationsimport pytest-from marimo._core import protocol+from marimo._core import backend as be, protocolfrom marimo._core.host import Hostfrom marimo._core.notebook import Notebookfrom marimo._core.stream import Event, Stream@@ -85,3 +85,53 @@ def test_readers_cannot_change_other_readers_or_replay() -> None:assert reread is not Noneassert isinstance(reread[0].data, protocol.ProjectMessage)assert reread[0].data.project == project()+++def test_deferred_events_join_history_before_delivery() -> None:+ stream = Stream()+ delivered: list[Event] = []+ stream.subscribe(delivered.append)++ with stream.deferred():+ stream.publish(project())+ assert [e.id for e in stream.since(0) or []] == [1]+ assert delivered == []++ assert [e.id for e in delivered] == [1]+++def test_subscribers_act_after_the_command_reaches_the_backend(+ host: Host, notebook: Notebook, backend: ScriptedBackend+) -> None:+ def stop_on_starting(event: Event) -> None:+ if event.name == "runtime":+ notebook.stop()++ host.subscribe(stop_on_starting, types={"runtime"})++ runtime, _ = notebook.start()++ assert backend.calls == [("start", runtime.id), ("stop",)]+ assert notebook.runtime is not None+ assert notebook.runtime.status == "terminating"+++def test_subscribers_cannot_interrupt_a_restart(+ host: Host, notebook: Notebook, backend: ScriptedBackend+) -> None:+ notebook.start()+ notebook.observe(be.Running())+ replacement = notebook.restart()+ backend.calls.clear()+ started: list[tuple[protocol.Runtime, bool]] = []++ def start_on_removal(event: Event) -> None:+ if event.name == "runtime.removed":+ started.append(notebook.start())++ host.subscribe(start_on_removal, types={"runtime"})++ notebook.observe(be.Exited())++ assert backend.calls == [("start", replacement.id)]+ assert started == [(replacement, False)]diff --git a/tests/_plugins/ui/_impl/dataframes/test_print_code.py b/tests/_plugins/ui/_impl/dataframes/test_print_code.pyindex 020419c19c..2b4dc6d3a6 100644--- a/tests/_plugins/ui/_impl/dataframes/test_print_code.py+++ b/tests/_plugins/ui/_impl/dataframes/test_print_code.py@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Literal, castimport narwhals.stable.v2 as nwimport pytest-from hypothesis import assume, given, settings, strategies as st+from hypothesis import assume, example, given, settings, strategies as stfrom marimo._dependencies.dependencies import DependencyManagerfrom marimo._plugins.ui._impl.dataframes.transforms.apply import (@@ -803,6 +803,14 @@ def test_print_code_contains_is_literal_pandas(value: str) -> None:df_size=3,))+@example(+ transform=SortColumnTransform(+ type=TransformType.SORT_COLUMN,+ column_id="dates",+ ascending=False,+ na_position="first",+ )+)@settings(deadline=None)@pytest.mark.skipif(not DependencyManager.polars.has(), reason="polars not installed"@@ -1035,6 +1043,16 @@ def test_print_code_result_matches_actual_transform_polars(code_result = code_result.sort(sortable_cols)real_result = real_result.sort(sortable_cols)+ if transform.type == TransformType.SORT_COLUMN:+ # Ties may differ between eager and lazy sorts. Check key order+ # before comparing rows using the fixture's unique string column.+ pl_testing.assert_series_equal(+ code_result[transform.column_id],+ real_result[transform.column_id],+ )+ code_result = code_result.sort("strings")+ real_result = real_result.sort("strings")+pl_testing.assert_frame_equal(code_result, real_result)diff --git a/tests/_plugins/ui/_impl/test_altair_chart.py b/tests/_plugins/ui/_impl/test_altair_chart.pyindex 759fdee481..75f8ade5c5 100644--- a/tests/_plugins/ui/_impl/test_altair_chart.py+++ b/tests/_plugins/ui/_impl/test_altair_chart.py@@ -773,17 +773,18 @@ def test_get_dataframe() -> None:@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")def test_get_dataframe_csv() -> None:import altair as alt- import pandas as pdimport polars as pldata = "https://cdn.jsdelivr.net/npm/[email protected]/data/stocks.csv"- fake_payload = b"symbol,price\nMSFT,39.81\nAAPL,28.13\n"+ expected = pl.DataFrame(+ {"symbol": ["MSFT", "AAPL"], "price": [39.81, 28.13]}+ )with unittest.mock.patch(- "urllib.request.urlopen",- return_value=io.BytesIO(fake_payload),- ):+ "polars.read_csv", return_value=expected+ ) as read_csv:chart = altair_chart(alt.Chart(data).mark_point().encode(x="values:Q"))- assert isinstance(chart.dataframe, (pd.DataFrame, pl.DataFrame))+ read_csv.assert_called_once_with(data)+ assert chart.dataframe is expected@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")