A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
refactor: replace bulk-mutation helpers with diff Transactions (#9797)
Kiran Gadhavea7bd5c62231c
diff --git a/marimo/_ast/app.py b/marimo/_ast/app.pyindex 9fef1a13a2..9c03ce1b28 100644--- a/marimo/_ast/app.py+++ b/marimo/_ast/app.py@@ -1052,7 +1052,8 @@ class InternalApp:config=config,cell=prev_compiled.get(cell_id),)- cm._replace_state_from(rebuilt)++ cm.apply_diff_from(rebuilt, source="cell-manager")return selfasync def run_cell_async(diff --git a/marimo/_ast/cell_manager.py b/marimo/_ast/cell_manager.pyindex 695a3bafb5..37c3b2d328 100644--- a/marimo/_ast/cell_manager.py+++ b/marimo/_ast/cell_manager.py@@ -23,7 +23,17 @@ from marimo._ast.models import CellDatafrom marimo._ast.names import DEFAULT_CELL_NAME, SETUP_CELL_NAMEfrom marimo._ast.parse import fixed_dedentfrom marimo._ast.pytest import process_for_pytest-from marimo._messaging.notebook.changes import CreateCell, Transaction+from marimo._messaging.notebook.changes import (+ CreateCell,+ DeleteCell,+ DocumentChange,+ ReorderCells,+ SetCode,+ SetConfig,+ SetName,+ Transaction,+ TransactionSource,+)from marimo._messaging.notebook.document import NotebookCell, NotebookDocumentfrom marimo._schemas.serialization import (CellDef,@@ -520,31 +530,6 @@ class CellManager:return nb_cell.idreturn None- def _replace_state_from(self, other: CellManager) -> None:- """Overwrite this manager's content with `other`'s, in place.-- Identity-preserving substitute for `self = other`: the- underlying `NotebookDocument` instance and the- `_compiled_cells` dict instance are kept; only their contents- are replaced. Any object holding a reference to this manager —- most notably the owning `Session` (which holds- `cell_manager.document` as `session.document`) — continues- to see live state without rebinding.-- `unparsable` is copied. `_cell_id_generator.seen_ids` is- unioned (never narrowed) so we don't recycle an id seen- previously even if it's gone from `other`.-- Transitional helper: cell-list bulk-replace bypasses- `NotebookDocument.apply`. Goes away once full-document- rebuilds are expressed as diff Transactions.- """- self._document._replace_cells(list(other._document._cells))- self._compiled_cells.clear()- self._compiled_cells.update(other._compiled_cells)- self.unparsable = other.unparsable- self._cell_id_generator.seen_ids |= other._cell_id_generator.seen_ids-def sort_cell_ids_by_similarity(self, prev_cell_manager: CellManager) -> None:@@ -568,6 +553,89 @@ class CellManager:for new_id in id_mapping:self._cell_id_generator.seen_ids.add(new_id)+ def _build_transaction(+ self, *, new: CellManager, source: TransactionSource+ ) -> tuple[Transaction, set[CellId_t]]:+ """Diff `self` -> `new`, returning `(transaction, changed_cell_ids)`.++ The transaction is unstamped; the caller applies it to the document+ (which assigns `version`). `changed_cell_ids` covers code, name,+ or config changes plus all creates and deletes - reorder-only cells+ are excluded.+ """+ prev_data = {cd.cell_id: cd for cd in self.cell_data()}+ prev_cell_ids = list(self.cell_ids())+ new_cell_ids = list(new.cell_ids())+ deleted = set(prev_data) - set(new_cell_ids)++ changes: list[DocumentChange] = []+ changed_cell_ids: set[CellId_t] = set(deleted)+ for cid in deleted:+ changes.append(DeleteCell(cell_id=cid))++ for cd in new.cell_data():+ prev_cd = prev_data.get(cd.cell_id)+ if prev_cd is None:+ changes.append(+ CreateCell(+ cell_id=cd.cell_id,+ code=cd.code,+ name=cd.name,+ config=cd.config,+ )+ )+ changed_cell_ids.add(cd.cell_id)+ continue+ if cd.code != prev_cd.code:+ changes.append(SetCode(cell_id=cd.cell_id, code=cd.code))+ changed_cell_ids.add(cd.cell_id)+ if cd.name != prev_cd.name:+ changes.append(SetName(cell_id=cd.cell_id, name=cd.name))+ changed_cell_ids.add(cd.cell_id)+ if cd.config != prev_cd.config:+ changes.append(+ SetConfig(+ cell_id=cd.cell_id,+ column=cd.config.column,+ disabled=cd.config.disabled,+ hide_code=cd.config.hide_code,+ )+ )+ changed_cell_ids.add(cd.cell_id)++ if tuple(new_cell_ids) != tuple(prev_cell_ids):+ changes.append(ReorderCells(cell_ids=tuple(new_cell_ids)))++ return (+ Transaction(changes=tuple(changes), source=source),+ changed_cell_ids,+ )++ def apply_diff_from(+ self, new: CellManager, *, source: TransactionSource+ ) -> tuple[Transaction, set[CellId_t]]:+ """Diff `self` -> `new`, apply to the document, and carry over the+ state the document doesn't track.++ `self` is mutated in place, so holders of it (and of its `document`)+ keep live state. `_compiled_cells`, `unparsable`, and `seen_ids`+ live outside the document, so they are copied alongside the+ transaction: `_compiled_cells` is mutated in place so its holders+ keep the dict, and `seen_ids` is unioned so a deleted id isn't reused.++ Returns the stamped transaction and the ids whose code, name, or+ config changed, plus all creates and deletes.+ """+ transaction, changed_cell_ids = self._build_transaction(+ new=new, source=source+ )+ transaction = self.document.apply(transaction)+ self._compiled_cells.clear()+ self._compiled_cells.update(new._compiled_cells)+ self.unparsable = new.unparsable+ self._cell_id_generator.seen_ids |= new._cell_id_generator.seen_ids+ return transaction, changed_cell_ids+@propertydef seen_ids(self) -> set[CellId_t]:return self._cell_id_generator.seen_idsdiff --git a/marimo/_messaging/notebook/document.py b/marimo/_messaging/notebook/document.pyindex 47fa91e130..87c4620c48 100644--- a/marimo/_messaging/notebook/document.py+++ b/marimo/_messaging/notebook/document.py@@ -237,35 +237,6 @@ class NotebookDocument:# Helpers# ------------------------------------------------------------------- def _replace_cells(self, cells: list[NotebookCell]) -> None:- """Bulk-replace the cell list in place, bypassing `apply`.-- Transitional. Used for full-document rebuilds (save round-trip)- until those flows are expressed as diff Transactions. The- `_cells` attribute is reassigned (not mutated element-wise)- so any external holder of the prior list keeps a snapshot of- pre-rebuild state — useful for diff comparison. Bumps- `version` so observers see the state change like they would- after `apply()`.-- Per-cell `version` is reconciled against the prior state: an id- whose code matches inherits the prior version (agent reads stay- valid); an id whose code changed gets `prior + 1` (invalidates- stale agent reads); brand-new ids keep the version they were- constructed with.- """- prior_by_id = {c.id: c for c in self._cells}- for i, c in enumerate(cells):- prior = prior_by_id.get(c.id)- if prior is None:- continue- if prior.code == c.code:- cells[i] = structs_replace(c, version=prior.version)- else:- cells[i] = structs_replace(c, version=prior.version + 1)- self._cells = cells- self._version += 1-def _rekey(self, mapping: dict[CellId_t, CellId_t]) -> None:"""Rekey cells by an old-id to new-id mapping, preserving order.diff --git a/marimo/_session/notebook/file_manager.py b/marimo/_session/notebook/file_manager.pyindex 26a97b9d02..5bf6df5fee 100644--- a/marimo/_session/notebook/file_manager.py+++ b/marimo/_session/notebook/file_manager.py@@ -12,13 +12,6 @@ from marimo._ast.app import App, InternalAppfrom marimo._ast.app_config import overloads_from_envfrom marimo._ast.cell import CellConfigfrom marimo._messaging.notebook.changes import (- CreateCell,- DeleteCell,- DocumentChange,- ReorderCells,- SetCode,- SetConfig,- SetName,Transaction,)from marimo._runtime.layout.layout import (@@ -43,7 +36,6 @@ LOGGER = _loggers.marimo_logger()if TYPE_CHECKING:from collections.abc import Sequence- from marimo._ast.cell_manager import CellManagerfrom marimo._messaging.notebook.document import NotebookCellfrom marimo._server.models.models import (CopyNotebookRequest,@@ -151,21 +143,8 @@ class AppFileManager:new_app = self._load_app(self.path)new_app.cell_manager.sort_cell_ids_by_similarity(current_cell_manager)- transaction, changed_cell_ids = _build_transaction(- prev=current_cell_manager, new=new_app.cell_manager- )- transaction = current_cell_manager.document.apply(transaction)-- # Carry over CellManager state that NotebookDocument doesn't track.- # clear()+update() preserves dict identity, matching the invariant- # established by CellManager._replace_state_from.- current_cell_manager._compiled_cells.clear()- current_cell_manager._compiled_cells.update(- new_app.cell_manager._compiled_cells- )- current_cell_manager.unparsable = new_app.cell_manager.unparsable- current_cell_manager._cell_id_generator.seen_ids |= (- new_app.cell_manager._cell_id_generator.seen_ids+ transaction, changed_cell_ids = current_cell_manager.apply_diff_from(+ new_app.cell_manager, source="file-watch")# Splice the preserved cell_manager into new_app, then swap the@@ -718,62 +697,3 @@ def _maybe_path(path: str | Path | None) -> Path | None:if isinstance(path, Path):return pathreturn Path(path)---def _build_transaction(- *, prev: CellManager, new: CellManager-) -> tuple[Transaction, set[CellId_t]]:- """Diff two CellManagers, returning `(transaction, changed_cell_ids)`.-- The transaction is unstamped; the caller applies it to the document- (which assigns `version`). `changed_cell_ids` covers code, name,- or config changes plus all creates and deletes — reorder-only cells- are excluded.- """- prev_data = {cd.cell_id: cd for cd in prev.cell_data()}- prev_cell_ids = list(prev.cell_ids())- new_cell_ids = list(new.cell_ids())- deleted = set(prev_data) - set(new_cell_ids)-- changes: list[DocumentChange] = []- changed_cell_ids: set[CellId_t] = set(deleted)- for cid in deleted:- changes.append(DeleteCell(cell_id=cid))-- for cd in new.cell_data():- prev_cd = prev_data.get(cd.cell_id)- if prev_cd is None:- changes.append(- CreateCell(- cell_id=cd.cell_id,- code=cd.code,- name=cd.name,- config=cd.config,- )- )- changed_cell_ids.add(cd.cell_id)- continue- if cd.code != prev_cd.code:- changes.append(SetCode(cell_id=cd.cell_id, code=cd.code))- changed_cell_ids.add(cd.cell_id)- if cd.name != prev_cd.name:- changes.append(SetName(cell_id=cd.cell_id, name=cd.name))- changed_cell_ids.add(cd.cell_id)- if cd.config != prev_cd.config:- changes.append(- SetConfig(- cell_id=cd.cell_id,- column=cd.config.column,- disabled=cd.config.disabled,- hide_code=cd.config.hide_code,- )- )- changed_cell_ids.add(cd.cell_id)-- if tuple(new_cell_ids) != tuple(prev_cell_ids):- changes.append(ReorderCells(cell_ids=tuple(new_cell_ids)))-- return (- Transaction(changes=tuple(changes), source="file-watch"),- changed_cell_ids,- )diff --git a/tests/_ast/test_app.py b/tests/_ast/test_app.pyindex ec9ec8482d..bfa0fc1516 100644--- a/tests/_ast/test_app.py+++ b/tests/_ast/test_app.py@@ -1825,6 +1825,21 @@ class TestInternalAppWithData:is original_compiled)+ def test_preserves_compiled_cells_dict_identity(self) -> None:+ internal_app, cell_id = self._seed_single()+ compiled = internal_app.cell_manager._compiled_cells++ internal_app.with_data(+ cell_ids=["new"],+ codes=["y = 2"],+ names=["__"],+ configs=[CellConfig()],+ )++ # The dict is mutated in place, not reassigned, so holders of the+ # original reference still see live state.+ assert internal_app.cell_manager._compiled_cells is compiled+def test_writes_new_codes_and_names(self) -> None:app = App()@@ -1848,6 +1863,110 @@ class TestInternalAppWithData:assert cd.code == "x = 99"assert cd.name == "renamed"+ def _seed_single(self) -> tuple[InternalApp, str]:+ app = App()++ @app.cell+ def _():+ x = 1+ return (x,)++ internal_app = InternalApp(app)+ cell_id = next(iter(internal_app.cell_manager.cell_ids()))+ return internal_app, cell_id++ def test_unchanged_code_preserves_cell_version(self) -> None:+ internal_app, cell_id = self._seed_single()+ doc = internal_app.cell_manager.document+ before = doc.get_cell_version(cell_id)++ internal_app.with_data(+ cell_ids=[cell_id],+ codes=[doc.get_cell(cell_id).code],+ names=["__"],+ configs=[CellConfig()],+ )++ assert doc.get_cell_version(cell_id) == before++ def test_changed_code_bumps_cell_version_by_one(self) -> None:+ internal_app, cell_id = self._seed_single()+ doc = internal_app.cell_manager.document+ before = doc.get_cell_version(cell_id)++ internal_app.with_data(+ cell_ids=[cell_id],+ codes=["x = 2"],+ names=["__"],+ configs=[CellConfig()],+ )++ assert doc.get_cell_version(cell_id) == before + 1++ def test_new_cell_keeps_constructed_version(self) -> None:+ internal_app, cell_id = self._seed_single()+ doc = internal_app.cell_manager.document++ internal_app.with_data(+ cell_ids=[cell_id, "new"],+ codes=["x = 1", "y = 2"],+ names=["__", "__"],+ configs=[CellConfig(), CellConfig()],+ )++ # New ids are created via CreateCell, which constructs a fresh cell+ # at version 0.+ assert doc.get_cell_version(CellId_t("new")) == 0++ def test_deleted_id_stays_reserved_for_generation(self) -> None:+ internal_app, cell_id = self._seed_single()++ # Drop the original cell.+ internal_app.with_data(+ cell_ids=["other"],+ codes=["y = 2"],+ names=["__"],+ configs=[CellConfig()],+ )++ # seen_ids remembers the dropped id so the generator won't mint it+ # again, even though it's gone from the document.+ seen = internal_app.cell_manager._cell_id_generator.seen_ids+ assert cell_id in seen++ def test_unparsable_not_carried_over(self) -> None:+ internal_app, cell_id = self._seed_single()+ internal_app.cell_manager.unparsable = True++ internal_app.with_data(+ cell_ids=[cell_id],+ codes=["x = 2"],+ names=["__"],+ configs=[CellConfig()],+ )++ # with_data rebuilds from parsable textual fields, so unparsable+ # resets to the rebuilt manager's value (False).+ assert internal_app.cell_manager.unparsable is False++ def test_noop_save_does_not_bump_document_version(self) -> None:+ # with_data routes through apply(); a save with identical content+ # produces an empty transaction, which leaves the document version+ # unchanged (consistent with SetCode no-op staying stable).+ internal_app, cell_id = self._seed_single()+ doc = internal_app.cell_manager.document+ cell = doc.get_cell(cell_id)+ before = doc.version++ internal_app.with_data(+ cell_ids=[cell_id],+ codes=[cell.code],+ names=[cell.name],+ configs=[cell.config],+ )++ assert doc.version == before+class TestInternalAppOverrides:"""Tests for InternalApp.overrides() method."""diff --git a/tests/_ast/test_cell_manager.py b/tests/_ast/test_cell_manager.pyindex f714be2b3b..2e8ebd9e03 100644--- a/tests/_ast/test_cell_manager.py+++ b/tests/_ast/test_cell_manager.py@@ -712,75 +712,6 @@ class TestGetCellDataView:assert cm.get_cell_data(CELL_A) is None-class TestReplaceStateFrom:- """`_replace_state_from` is the in-place substitute for- `self = other`. It preserves identity of `_document` and- `_compiled_cells` so external holders of those references — most- notably the owning `Session` — see the new state without any- rebinding.- """-- def test_preserves_document_identity(self) -> None:- cm = CellManager()- cm.register_cell(CELL_A, "x = 1", CellConfig())- doc = cm.document-- other = CellManager()- other.register_cell(CELL_B, "y = 2", CellConfig())- cm._replace_state_from(other)-- assert cm.document is doc-- def test_preserves_compiled_cells_dict_identity(self) -> None:- cm = CellManager()- cm.register_cell(CELL_A, "x = 1", CellConfig())- compiled = cm._compiled_cells-- other = CellManager()- other.register_cell(CELL_B, "y = 2", CellConfig())- cm._replace_state_from(other)-- assert cm._compiled_cells is compiled-- def test_replaces_cell_list_contents(self) -> None:- cm = CellManager()- cm.register_cell(CELL_A, "x = 1", CellConfig())-- other = CellManager()- other.register_cell(CELL_B, "y = 2", CellConfig())- other.register_cell(CELL_C, "z = 3", CellConfig())- cm._replace_state_from(other)-- assert list(cm.cell_ids()) == [CELL_B, CELL_C]- assert cm.get_cell_code(CELL_A) is None-- def test_unions_seen_ids_no_narrowing(self) -> None:- cm = CellManager()- cm.seen_ids.add(CELL_A)- # CELL_A is in cm.seen_ids; CELL_B is not.-- other = CellManager()- other.seen_ids.add(CELL_B)- # other.seen_ids has CELL_B but not CELL_A.-- cm._replace_state_from(other)-- # Union: both are remembered. Generating new ids later should- # not collide because seen_ids guards against id reuse.- assert CELL_A in cm.seen_ids- assert CELL_B in cm.seen_ids-- def test_carries_unparsable_flag(self) -> None:- cm = CellManager()- assert cm.unparsable is False-- other = CellManager()- other.unparsable = True- cm._replace_state_from(other)-- assert cm.unparsable is True--class TestSortCellIdsByCompiledCells:"""`sort_cell_ids_by_similarity` rekeys `_compiled_cells` inlockstep with the document's cell ids. This matters because lookupsdiff --git a/tests/_code_mode/test_staleness.py b/tests/_code_mode/test_staleness.pyindex e6fa5be4ab..0540854dc1 100644--- a/tests/_code_mode/test_staleness.py+++ b/tests/_code_mode/test_staleness.py@@ -178,50 +178,34 @@ class TestCrossContextWriteRead:class TestFileWatchReload:- async def test_replace_cells_same_code_keeps_reads_valid(- self, k: Kernel- ) -> None:- from marimo._ast.cell import CellConfig-+ async def test_same_code_keeps_reads_valid(self, k: Kernel) -> None:await _seed(k, "0", "a = 1")with _ctx(k) as ctx:async with ctx as nb:_ = nb.cells["0"].codewith _ctx(k) as ctx:- ctx._document._replace_cells(- [- NotebookCell(- id=CellId_t("0"),- code="a = 1",- name="",- config=CellConfig(),- )- ]+ ctx._document.apply(+ Transaction(+ changes=(SetCode(cell_id=CellId_t("0"), code="a = 1"),),+ source="file-watch",+ ))async with ctx as nb:nb.edit_cell("0", "a = 2")- async def test_replace_cells_changed_code_invalidates_reads(- self, k: Kernel- ) -> None:- from marimo._ast.cell import CellConfig-+ async def test_changed_code_invalidates_reads(self, k: Kernel) -> None:await _seed(k, "0", "a = 1")with _ctx(k) as ctx:async with ctx as nb:_ = nb.cells["0"].codewith _ctx(k) as ctx:- ctx._document._replace_cells(- [- NotebookCell(- id=CellId_t("0"),- code="a = 100",- name="",- config=CellConfig(),- )- ]+ ctx._document.apply(+ Transaction(+ changes=(SetCode(cell_id=CellId_t("0"), code="a = 100"),),+ source="file-watch",+ ))async with ctx as nb:with pytest.raises(StaleCellError):diff --git a/tests/_messaging/notebook/test_document.py b/tests/_messaging/notebook/test_document.pyindex 8a77581ee3..0aa7e78d68 100644--- a/tests/_messaging/notebook/test_document.py+++ b/tests/_messaging/notebook/test_document.py@@ -323,53 +323,6 @@ class TestCellVersion:doc._rekey({CellId_t("a"): CellId_t("x")})assert doc.get_cell_version(CellId_t("x")) == 2- def test_replace_cells_same_code_preserves_version(self) -> None:- doc = _doc("a")- doc.apply(_tx(SetCode(cell_id=CellId_t("a"), code="x")))- assert doc.get_cell_version(CellId_t("a")) == 1- doc._replace_cells(- [- NotebookCell(- id=CellId_t("a"),- code="x",- name="__",- config=CellConfig(),- )- ]- )- assert doc.get_cell_version(CellId_t("a")) == 1-- def test_replace_cells_changed_code_bumps_version(self) -> None:- doc = _doc("a")- doc.apply(_tx(SetCode(cell_id=CellId_t("a"), code="x")))- assert doc.get_cell_version(CellId_t("a")) == 1- doc._replace_cells(- [- NotebookCell(- id=CellId_t("a"),- code="y",- name="__",- config=CellConfig(),- )- ]- )- assert doc.get_cell_version(CellId_t("a")) == 2-- def test_replace_cells_new_id_keeps_constructed_version(self) -> None:- doc = _doc("a")- doc._replace_cells(- [- NotebookCell(- id=CellId_t("b"),- code="y",- name="__",- config=CellConfig(),- version=42,- )- ]- )- assert doc.get_cell_version(CellId_t("b")) == 42-# ------------------------------------------------------------------# SetName@@ -562,22 +515,6 @@ class TestVersion:assert doc.version == 1assert applied.version == 1- def test_replace_cells_bumps_version(self) -> None:- doc = _doc("a", "b")- starting = doc.version- doc._replace_cells([_cell("c"), _cell("d")])- assert doc.version == starting + 1-- def test_replace_cells_preserves_document_identity(self) -> None:- doc = _doc("a")- new_cells = [_cell("b"), _cell("c")]- doc._replace_cells(new_cells)- assert _ids(doc) == ["b", "c"]- # Reassigning the cells list — not mutating in place — lets prior- # holders (e.g. file-watch diff path) keep a snapshot of the- # pre-rebuild state for comparison.- assert doc._cells is new_cells-def test_rekey_bumps_version(self) -> None:doc = _doc("a", "b")starting = doc.versiondiff --git a/tests/_session/test_file_change_handler.py b/tests/_session/test_file_change_handler.pyindex e6d4f6cc6a..b872610137 100644--- a/tests/_session/test_file_change_handler.py+++ b/tests/_session/test_file_change_handler.py@@ -35,7 +35,6 @@ from marimo._session.file_change_handler import (RunModeReloadStrategy,)from marimo._session.notebook import AppFileManager-from marimo._session.notebook.file_manager import _build_transactionfrom marimo._types.ids import CellId_tif TYPE_CHECKING:@@ -131,8 +130,9 @@ def _run_reload(mock_session.document = prev_documentstrategy = EditModeReloadStrategy(config_manager)cell_ids = list(afm.app.cell_manager.cell_ids())- transaction, _ = _build_transaction(- prev=_cm_from_doc(prev_document), new=afm.app.cell_manager+ transaction, _ = _cm_from_doc(prev_document)._build_transaction(+ new=afm.app.cell_manager,+ source="file-watch",)strategy.handle_reload(mock_session,@@ -259,8 +259,9 @@ def test_edit_mode_reload_with_deleted_cells(mock_session.document = prev_documentstrategy = EditModeReloadStrategy(config)- transaction, _ = _build_transaction(- prev=_cm_from_doc(prev_document), new=afm.app.cell_manager+ transaction, _ = _cm_from_doc(prev_document)._build_transaction(+ new=afm.app.cell_manager,+ source="file-watch",)strategy.handle_reload(mock_session,