Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

Deliver a command's events after its backend call (#11205)

Trevor Manz9b14875ae48f
diff --git a/marimo/_core/notebook.py b/marimo/_core/notebook.py
index f9b13c8177..c4e4d91a5e 100644
--- a/marimo/_core/notebook.py
+++ b/marimo/_core/notebook.py
@@ -25,13 +25,17 @@ if TYPE_CHECKING:
class Notebook:
- """A notebook's state and commands.
+ """A document, its results, and the runtime that produces them.
- Snapshots preserve insertion order for runs and executions, and for
- outputs within each execution.
- Current cell results survive runtime stops and restarts. The host
- chooses how long to retain completed ephemeral results.
- Removing the notebook closes it; further commands raise `NotFound`.
+ Clients attach to a notebook, edit it, and run it. Every change is
+ published as an event so that each client sees the same notebook.
+ Commands change it and call the backend, which reports back through
+ `observe`.
+
+ Subscribers hear about a command once it has reached the backend.
+ Results survive runtime stops and restarts, and the host decides how
+ long completed scratch results are kept. Commands on a notebook the
+ host has removed raise `NotFound`.
"""
def __init__(
@@ -86,14 +90,15 @@ class Notebook:
Returns the runtime and whether a new runtime was created.
An existing runtime is returned unchanged, regardless of status.
"""
- self._require_open()
- if self._runtime is not None:
- return self._runtime, False
- runtime = self._new_runtime("starting", None, None)
- self._lifecycle_request = request_id
- self._set_runtime(runtime)
- self._backend.start(runtime.id)
- return runtime, True
+ with self._stream.deferred():
+ self._require_open()
+ if self._runtime is not None:
+ return self._runtime, False
+ runtime = self._new_runtime("starting", None, None)
+ self._lifecycle_request = request_id
+ self._set_runtime(runtime)
+ self._backend.start(runtime.id)
+ return runtime, True
def stop(self, *, request_id: str | None = None) -> protocol.Runtime:
"""Stop the runtime for all attached clients, retaining its results.
@@ -105,19 +110,20 @@ class Notebook:
Raises:
NoRuntime: If the notebook has no runtime.
"""
- self._require_open()
- runtime = self._require_runtime()
- self._lifecycle_request = request_id
- self._replacement = None
- if runtime.status == "terminating":
- return runtime
- terminating = replace(runtime, status="terminating")
- self._set_runtime(terminating)
- if runtime.status == "failed":
- self._end_runtime()
- else:
- self._backend.stop()
- return terminating
+ with self._stream.deferred():
+ self._require_open()
+ runtime = self._require_runtime()
+ self._lifecycle_request = request_id
+ self._replacement = None
+ if runtime.status == "terminating":
+ return runtime
+ terminating = replace(runtime, status="terminating")
+ self._set_runtime(terminating)
+ if runtime.status == "failed":
+ self._end_runtime()
+ else:
+ self._backend.stop()
+ return terminating
def restart(self, *, request_id: str | None = None) -> protocol.Runtime:
"""Stop the runtime and start a replacement with a new ID.
@@ -129,19 +135,20 @@ class Notebook:
NoRuntime: If the notebook has no runtime.
Conflict: If the runtime is not running.
"""
- self._require_open()
- runtime = self._require_runtime()
- if runtime.status != "running":
- raise Conflict(
- f"the runtime is {runtime.status}; only a running one restarts"
- )
- self._lifecycle_request = request_id
- replacement = self._new_runtime("starting", None, None)
- self._replacement = replacement
- terminating = replace(runtime, status="terminating")
- self._set_runtime(terminating)
- self._backend.stop()
- return replacement
+ with self._stream.deferred():
+ self._require_open()
+ runtime = self._require_runtime()
+ if runtime.status != "running":
+ raise Conflict(
+ f"the runtime is {runtime.status}; only a running one restarts"
+ )
+ self._lifecycle_request = request_id
+ replacement = self._new_runtime("starting", None, None)
+ self._replacement = replacement
+ terminating = replace(runtime, status="terminating")
+ self._set_runtime(terminating)
+ self._backend.stop()
+ return replacement
def _require_runtime(self) -> protocol.Runtime:
if self._runtime is None:
@@ -175,24 +182,25 @@ class Notebook:
NoRuntime: If the notebook has no runtime.
Conflict: If the runtime is not running, or a run is in progress.
"""
- self._require_open()
- runtime = self._require_runtime()
- if runtime.status != "running":
- raise Conflict(f"the runtime is {runtime.status}.")
- if (active := self._state.active_run) is not None:
- raise Conflict(f"run {active.id} is in progress.")
- run = protocol.Run(
- id=mint(protocol.RunId, "run"),
- notebook_id=self.id,
- caused_by=None,
- status="queued",
- outcome=None,
- started_at=None,
- completed_at=None,
- )
- self._state.add_run(run, request_id)
- self._backend.run(run.id, code)
- return run
+ with self._stream.deferred():
+ self._require_open()
+ runtime = self._require_runtime()
+ if runtime.status != "running":
+ raise Conflict(f"the runtime is {runtime.status}.")
+ if (active := self._state.active_run) is not None:
+ raise Conflict(f"run {active.id} is in progress.")
+ run = protocol.Run(
+ id=mint(protocol.RunId, "run"),
+ notebook_id=self.id,
+ caused_by=None,
+ status="queued",
+ outcome=None,
+ started_at=None,
+ completed_at=None,
+ )
+ self._state.add_run(run, request_id)
+ self._backend.run(run.id, code)
+ return run
def interrupt(self, run_id: str) -> protocol.Run:
"""Request interruption of a run in progress.
@@ -279,11 +287,11 @@ class Notebook:
if runtime.status == "terminating" and self._replacement is not None:
replacement, self._replacement = self._replacement, None
request_id = self._lifecycle_request
- self._end_runtime()
- self._lifecycle_request = request_id
- self._set_runtime(replacement)
- assert self._runtime is not None
- self._backend.start(self._runtime.id)
+ with self._stream.deferred():
+ self._end_runtime()
+ self._lifecycle_request = request_id
+ self._set_runtime(replacement)
+ self._backend.start(replacement.id)
elif runtime.status == "terminating":
self._end_runtime()
else:
@@ -317,7 +325,10 @@ class Notebook:
# ---- stream --------------------------------------------------------
def _snapshot(self) -> list[Event]:
- """Return notebook objects as snapshot events, parents first."""
+ """Return notebook objects as snapshot events, parents first.
+
+ Runs, executions, and outputs keep their insertion order.
+ """
bodies: list[protocol.Runtime | protocol.Attachment] = []
if self._runtime is not None:
bodies.append(self._runtime)
diff --git a/marimo/_core/stream.py b/marimo/_core/stream.py
index cbb545b7c2..310a0d5ea7 100644
--- a/marimo/_core/stream.py
+++ b/marimo/_core/stream.py
@@ -9,7 +9,8 @@ from __future__ import annotations
import logging
from collections import deque
-from collections.abc import Callable, Collection
+from collections.abc import Callable, Collection, Iterator
+from contextlib import contextmanager
from copy import deepcopy
from dataclasses import dataclass
from datetime import datetime, timezone
@@ -206,6 +207,23 @@ class Stream:
return unsubscribe
+ @contextmanager
+ def deferred(self) -> Iterator[None]:
+ """Deliver events published in the block once it ends.
+
+ Events still join the history at once, so cursors and replay are
+ unaffected. Nested blocks defer to the outermost one.
+ """
+ if self._dispatching:
+ yield
+ return
+ self._dispatching = True
+ try:
+ yield
+ finally:
+ self._dispatching = False
+ self._drain()
+
def _append(self, event: Event) -> Event:
self._last_id += 1
stamped = Event(
@@ -218,15 +236,18 @@ class Stream:
self._history.append(stamped)
self._pending.append((stamped, tuple(self._listeners)))
if not self._dispatching:
- self._dispatching = True
- try:
- while self._pending:
- pending, listeners = self._pending.popleft()
- for listener in listeners:
- try:
- listener(deepcopy(pending))
- except Exception:
- LOGGER.exception("Error delivering host event")
- finally:
- self._dispatching = False
+ self._drain()
return deepcopy(stamped)
+
+ def _drain(self) -> None:
+ self._dispatching = True
+ try:
+ while self._pending:
+ pending, listeners = self._pending.popleft()
+ for listener in listeners:
+ try:
+ listener(deepcopy(pending))
+ except Exception:
+ LOGGER.exception("Error delivering host event")
+ finally:
+ self._dispatching = False
diff --git a/tests/_core/test_stream.py b/tests/_core/test_stream.py
index 6cd53055cc..25d1c8936b 100644
--- a/tests/_core/test_stream.py
+++ b/tests/_core/test_stream.py
@@ -5,7 +5,7 @@ from __future__ import annotations
import pytest
-from marimo._core import protocol
+from marimo._core import backend as be, protocol
from marimo._core.host import Host
from marimo._core.notebook import Notebook
from marimo._core.stream import Event, Stream
@@ -85,3 +85,53 @@ def test_readers_cannot_change_other_readers_or_replay() -> None:
assert reread is not None
assert isinstance(reread[0].data, protocol.ProjectMessage)
assert reread[0].data.project == project()
+
+
+def test_deferred_events_join_history_before_delivery() -> None:
+ stream = Stream()
+ delivered: list[Event] = []
+ stream.subscribe(delivered.append)
+
+ with stream.deferred():
+ stream.publish(project())
+ assert [e.id for e in stream.since(0) or []] == [1]
+ assert delivered == []
+
+ assert [e.id for e in delivered] == [1]
+
+
+def test_subscribers_act_after_the_command_reaches_the_backend(
+ host: Host, notebook: Notebook, backend: ScriptedBackend
+) -> None:
+ def stop_on_starting(event: Event) -> None:
+ if event.name == "runtime":
+ notebook.stop()
+
+ host.subscribe(stop_on_starting, types={"runtime"})
+
+ runtime, _ = notebook.start()
+
+ assert backend.calls == [("start", runtime.id), ("stop",)]
+ assert notebook.runtime is not None
+ assert notebook.runtime.status == "terminating"
+
+
+def test_subscribers_cannot_interrupt_a_restart(
+ host: Host, notebook: Notebook, backend: ScriptedBackend
+) -> None:
+ notebook.start()
+ notebook.observe(be.Running())
+ replacement = notebook.restart()
+ backend.calls.clear()
+ started: list[tuple[protocol.Runtime, bool]] = []
+
+ def start_on_removal(event: Event) -> None:
+ if event.name == "runtime.removed":
+ started.append(notebook.start())
+
+ host.subscribe(start_on_removal, types={"runtime"})
+
+ notebook.observe(be.Exited())
+
+ assert backend.calls == [("start", replacement.id)]
+ assert started == [(replacement, False)]
diff --git a/tests/_plugins/ui/_impl/dataframes/test_print_code.py b/tests/_plugins/ui/_impl/dataframes/test_print_code.py
index 020419c19c..2b4dc6d3a6 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_print_code.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_print_code.py
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Literal, cast
import narwhals.stable.v2 as nw
import pytest
-from hypothesis import assume, given, settings, strategies as st
+from hypothesis import assume, example, given, settings, strategies as st
from marimo._dependencies.dependencies import DependencyManager
from marimo._plugins.ui._impl.dataframes.transforms.apply import (
@@ -803,6 +803,14 @@ def test_print_code_contains_is_literal_pandas(value: str) -> None:
df_size=3,
)
)
+@example(
+ transform=SortColumnTransform(
+ type=TransformType.SORT_COLUMN,
+ column_id="dates",
+ ascending=False,
+ na_position="first",
+ )
+)
@settings(deadline=None)
@pytest.mark.skipif(
not DependencyManager.polars.has(), reason="polars not installed"
@@ -1035,6 +1043,16 @@ def test_print_code_result_matches_actual_transform_polars(
code_result = code_result.sort(sortable_cols)
real_result = real_result.sort(sortable_cols)
+ if transform.type == TransformType.SORT_COLUMN:
+ # Ties may differ between eager and lazy sorts. Check key order
+ # before comparing rows using the fixture's unique string column.
+ pl_testing.assert_series_equal(
+ code_result[transform.column_id],
+ real_result[transform.column_id],
+ )
+ code_result = code_result.sort("strings")
+ real_result = real_result.sort("strings")
+
pl_testing.assert_frame_equal(code_result, real_result)
diff --git a/tests/_plugins/ui/_impl/test_altair_chart.py b/tests/_plugins/ui/_impl/test_altair_chart.py
index 759fdee481..75f8ade5c5 100644
--- a/tests/_plugins/ui/_impl/test_altair_chart.py
+++ b/tests/_plugins/ui/_impl/test_altair_chart.py
@@ -773,17 +773,18 @@ def test_get_dataframe() -> None:
@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")
def test_get_dataframe_csv() -> None:
import altair as alt
- import pandas as pd
import polars as pl
data = "https://cdn.jsdelivr.net/npm/[email protected]/data/stocks.csv"
- fake_payload = b"symbol,price\nMSFT,39.81\nAAPL,28.13\n"
+ expected = pl.DataFrame(
+ {"symbol": ["MSFT", "AAPL"], "price": [39.81, 28.13]}
+ )
with unittest.mock.patch(
- "urllib.request.urlopen",
- return_value=io.BytesIO(fake_payload),
- ):
+ "polars.read_csv", return_value=expected
+ ) as read_csv:
chart = altair_chart(alt.Chart(data).mark_point().encode(x="values:Q"))
- assert isinstance(chart.dataframe, (pd.DataFrame, pl.DataFrame))
+ read_csv.assert_called_once_with(data)
+ assert chart.dataframe is expected
@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")