Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: More aggressive file reconciliation in `--watch` (#6701)

Dylan Madisettibb09501dbb43
diff --git a/docs/guides/editor_features/watching.md b/docs/guides/editor_features/watching.md
index f5a63a81f2..91e6bcdb36 100644
--- a/docs/guides/editor_features/watching.md
+++ b/docs/guides/editor_features/watching.md
@@ -11,6 +11,9 @@ have the changes automatically reflected in your browser.
For better performance, install [watchdog](https://pypi.org/project/watchdog/).
Without watchdog, marimo resorts to polling.
+!!! warn "Watch works best with autosave"
+ Verify your settings in `User Settings` > `Editor` > `Autosave` > `Autosave enabled`
+
## marimo's file format
!!! tip "File format tutorial"
diff --git a/marimo/_runtime/requests.py b/marimo/_runtime/requests.py
index 290a546d8e..726f05838c 100644
--- a/marimo/_runtime/requests.py
+++ b/marimo/_runtime/requests.py
@@ -181,6 +181,29 @@ class ExecuteMultipleRequest(msgspec.Struct, rename="camel"):
)
+class SyncGraphRequest(msgspec.Struct, rename="camel"):
+ # ids of cells known to filemanager
+ cells: dict[CellId_t, str]
+ # From the list of ALL cells that filemanager knows about,
+ # denote what should be run/ updated or deleted.
+ run_ids: list[CellId_t]
+ delete_ids: list[CellId_t]
+ # time at which the request was received
+ timestamp: float = msgspec.field(default_factory=time.time)
+
+ @property
+ def execution_requests(self) -> list[ExecutionRequest]:
+ return [
+ ExecutionRequest(
+ cell_id=cell_id,
+ code=self.cells[cell_id],
+ request=None,
+ timestamp=self.timestamp,
+ )
+ for cell_id in self.run_ids
+ ]
+
+
class ExecuteScratchpadRequest(msgspec.Struct, rename="camel"):
code: str
# incoming request, e.g. from Starlette or FastAPI
@@ -420,6 +443,7 @@ ControlRequest = Union[
DeleteCellRequest,
PdbRequest,
ExecuteScratchpadRequest,
+ SyncGraphRequest,
RenameRequest,
SetCellConfigRequest,
SetUserConfigRequest,
diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.py
index d49115f285..5e47a180df 100644
--- a/marimo/_runtime/runtime.py
+++ b/marimo/_runtime/runtime.py
@@ -154,6 +154,7 @@ from marimo._runtime.requests import (
SetUIElementValueRequest,
SetUserConfigRequest,
StopRequest,
+ SyncGraphRequest,
ValidateSQLRequest,
)
from marimo._runtime.runner import cell_runner
@@ -1486,6 +1487,50 @@ class Kernel:
)
)
+ @kernel_tracer.start_as_current_span("sync_graph")
+ async def sync_graph(
+ self,
+ cells: dict[CellId_t, str],
+ run_ids: list[CellId_t],
+ delete_ids: list[CellId_t],
+ ) -> None:
+ """Synchronize kernel graph with file manager state.
+
+ File manager is the source of truth after a reload. This method
+ ensures the kernel graph matches file manager's state by:
+ 1. Deleting cells that file manager doesn't know about (orphaned)
+ 2. Deleting cells explicitly marked for deletion
+ 3. Running/updating cells that changed
+
+ Args:
+ cells: All cells known to file manager (cell_id -> code)
+ run_ids: Cell IDs that should be executed/updated
+ delete_ids: Cell IDs that should be deleted
+ """
+ # Find orphaned cells: in graph but not known to file manager
+ orphaned_cells = set(self.graph.cells.keys()) - set(cells.keys())
+ all_delete_ids = set(delete_ids) | orphaned_cells
+
+ # Create execution requests for cells to run
+ execution_requests = [
+ ExecutionRequest(cell_id=cell_id, code=cells[cell_id])
+ for cell_id in run_ids
+ ]
+
+ # Create deletion requests for all cells to delete
+ deletion_requests = [
+ DeleteCellRequest(cell_id=cell_id) for cell_id in all_delete_ids
+ ]
+
+ # Clean up uninstantiated requests for deleted cells
+ for cell_id in all_delete_ids:
+ if cell_id in self._uninstantiated_execution_requests:
+ del self._uninstantiated_execution_requests[cell_id]
+
+ # Use existing mutate_graph infrastructure to update the graph
+ self.mutate_graph(execution_requests, deletion_requests)
+ await self.run(execution_requests)
+
@kernel_tracer.start_as_current_span("run")
async def run(
self, execution_requests: Sequence[ExecutionRequest]
@@ -2144,6 +2189,15 @@ class Kernel:
await self.run(request.execution_requests)
CompletedRun().broadcast()
+ async def handle_sync_graph(
+ request: SyncGraphRequest,
+ ) -> None:
+ with http_request_context(None):
+ await self.sync_graph(
+ request.cells, request.run_ids, request.delete_ids
+ )
+ CompletedRun().broadcast()
+
async def handle_execute_scratchpad(
request: ExecuteScratchpadRequest,
) -> None:
@@ -2205,6 +2259,7 @@ class Kernel:
handler.register(CreationRequest, handle_instantiate)
handler.register(DeleteCellRequest, self.delete_cell)
handler.register(ExecuteMultipleRequest, handle_execute_multiple)
+ handler.register(SyncGraphRequest, handle_sync_graph)
handler.register(ExecuteScratchpadRequest, handle_execute_scratchpad)
handler.register(ExecuteStaleRequest, handle_execute_stale)
handler.register(FunctionCallRequest, handle_function_call)
diff --git a/marimo/_server/file_manager.py b/marimo/_server/file_manager.py
index 16e55685b3..29dca0a8cf 100644
--- a/marimo/_server/file_manager.py
+++ b/marimo/_server/file_manager.py
@@ -67,6 +67,7 @@ class AppFileManager:
# Return the changes cell IDs
prev_cell_ids = set(prev_cell_manager.cell_ids())
current_cell_ids = set(self.app.cell_manager.cell_ids())
+
# Capture deleted cells
changed_cell_ids: set[CellId_t] = prev_cell_ids - current_cell_ids
for cell_id in current_cell_ids:
diff --git a/marimo/_server/session/session_view.py b/marimo/_server/session/session_view.py
index 29c5dfd7aa..f21957b52e 100644
--- a/marimo/_server/session/session_view.py
+++ b/marimo/_server/session/session_view.py
@@ -30,6 +30,7 @@ from marimo._runtime.requests import (
ExecuteMultipleRequest,
ExecutionRequest,
SetUIElementValueRequest,
+ SyncGraphRequest,
)
from marimo._sql.engines.duckdb import INTERNAL_DUCKDB_ENGINE
from marimo._types.ids import CellId_t, WidgetModelId
@@ -126,7 +127,7 @@ class SessionView:
if isinstance(request, SetUIElementValueRequest):
for object_id, value in request.ids_and_values:
self._add_ui_value(object_id, value)
- elif isinstance(request, ExecuteMultipleRequest):
+ elif isinstance(request, (ExecuteMultipleRequest, SyncGraphRequest)):
for execution_request in request.execution_requests:
self._add_last_run_code(execution_request)
elif isinstance(request, CreationRequest):
diff --git a/marimo/_server/sessions.py b/marimo/_server/sessions.py
index fa1c2214d2..32d3d72ede 100644
--- a/marimo/_server/sessions.py
+++ b/marimo/_server/sessions.py
@@ -57,6 +57,7 @@ from marimo._runtime.requests import (
SerializedCLIArgs,
SerializedQueryParams,
SetUIElementValueRequest,
+ SyncGraphRequest,
)
from marimo._server.exceptions import InvalidSessionException
from marimo._server.file_manager import AppFileManager
@@ -579,19 +580,25 @@ class Session:
if isinstance(request, SetUIElementValueRequest):
self._queue_manager.set_ui_element_queue.put(request)
# Propagate the control request to the room
- if isinstance(request, ExecuteMultipleRequest):
+ if isinstance(request, (ExecuteMultipleRequest, SyncGraphRequest)):
+ if isinstance(request, ExecuteMultipleRequest):
+ cell_ids = request.cell_ids
+ codes = request.codes
+ else:
+ cell_ids = request.run_ids
+ codes = [request.cells[cell_id] for cell_id in cell_ids]
self.room.broadcast(
UpdateCellCodes(
- cell_ids=request.cell_ids,
- codes=request.codes,
+ cell_ids=cell_ids,
+ codes=codes,
# Not stale because we just ran the code
code_is_stale=False,
),
except_consumer=from_consumer_id,
)
- if len(request.cell_ids) == 1:
+ if len(cell_ids) == 1:
self.room.broadcast(
- FocusCell(cell_id=request.cell_ids[0]),
+ FocusCell(cell_id=cell_ids[0]),
except_consumer=from_consumer_id,
)
self.session_view.add_control_request(request)
@@ -1203,40 +1210,29 @@ class SessionFileChangeHandler:
"watcher_on_save"
]
should_autorun = watcher_on_save == "autorun"
+ deleted = {
+ cell_id for cell_id in changed_cell_ids if cell_id not in cell_ids
+ }
# Auto-run cells if configured
if should_autorun:
- cell_ids_to_idx = {
- cell_id: idx for idx, cell_id in enumerate(cell_ids)
- }
- deleted = {
- cell_id
- for cell_id in changed_cell_ids
- if cell_id not in cell_ids_to_idx
- }
changed_cell_ids_list = list(changed_cell_ids - deleted)
- changed_codes = [
- codes[cell_ids_to_idx[cell_id]]
- for cell_id in changed_cell_ids_list
- if cell_id not in deleted
- ]
+ cells = dict(zip(cell_ids, codes))
- if changed_cell_ids_list:
- # This runs the request and also runs UpdateCellCodes
- session.put_control_request(
- ExecuteMultipleRequest(
- cell_ids=changed_cell_ids_list,
- codes=changed_codes,
- request=None,
- ),
- from_consumer_id=None,
- )
+ session.put_control_request(
+ SyncGraphRequest(
+ cells=cells,
+ run_ids=changed_cell_ids_list,
+ delete_ids=list(deleted),
+ ),
+ from_consumer_id=None,
+ )
+ else:
for to_delete in deleted:
session.put_control_request(
DeleteCellRequest(cell_id=to_delete),
from_consumer_id=None,
)
- else:
session.write_operation(
UpdateCellCodes(
cell_ids=cell_ids,
diff --git a/tests/_runtime/test_runtime.py b/tests/_runtime/test_runtime.py
index e381905cca..96d18cfba5 100644
--- a/tests/_runtime/test_runtime.py
+++ b/tests/_runtime/test_runtime.py
@@ -1650,6 +1650,235 @@ except NameError:
else:
assert cell_op.run_id is not None
+ async def test_sync_graph_basic(self, execution_kernel: Kernel) -> None:
+ """Test basic synchronization: file changes cell B in A→B→C chain.
+
+ Uses execution_kernel (reactive mode) to test cascade behavior.
+ """
+ k = execution_kernel
+ # Setup: Create initial graph state with dependencies
+ await k.run(
+ [
+ ExecutionRequest(cell_id="0", code="x = 1"),
+ ExecutionRequest(cell_id="1", code="y = x + 1"),
+ ExecutionRequest(cell_id="2", code="z = y + 1"),
+ ]
+ )
+ assert k.globals["x"] == 1
+ assert k.globals["y"] == 2
+ assert k.globals["z"] == 3
+ assert len(k.graph.cells) == 3
+
+ # Action: Sync with cell 1 changed
+ await k.sync_graph(
+ cells={"0": "x = 1", "1": "y = x + 10", "2": "z = y + 1"},
+ run_ids=["1"],
+ delete_ids=[],
+ )
+
+ # Verify: Cell 1 and its descendant (2) are re-executed
+ assert k.globals["x"] == 1 # unchanged
+ assert k.globals["y"] == 11 # updated
+ assert k.globals["z"] == 12 # cascaded from y
+
+ async def test_sync_graph_orphaned_cells(self, any_kernel: Kernel) -> None:
+ """Test orphaned cell detection: cell in kernel but not file manager."""
+ k = any_kernel
+ # Setup: Create 3 cells
+ await k.run(
+ [
+ ExecutionRequest(cell_id="0", code="x = 1"),
+ ExecutionRequest(cell_id="1", code="y = x + 1"),
+ ExecutionRequest(cell_id="2", code="z = y + 1"),
+ ]
+ )
+ assert "z" in k.globals
+ assert len(k.graph.cells) == 3
+
+ # Action: Sync with file manager only knowing about cells 0 and 1
+ # Cell 2 is orphaned (deleted from file)
+ await k.sync_graph(
+ cells={"0": "x = 1", "1": "y = x + 1"},
+ run_ids=[],
+ delete_ids=[],
+ )
+
+ # Verify: Cell 2 is automatically deleted
+ assert len(k.graph.cells) == 2
+ assert "z" not in k.globals
+ assert "0" in k.graph.cells
+ assert "1" in k.graph.cells
+ assert "2" not in k.graph.cells
+
+ async def test_sync_graph_combined_orphan_and_explicit_delete(
+ self, any_kernel: Kernel
+ ) -> None:
+ """Test both orphaned and explicitly deleted cells."""
+ k = any_kernel
+ # Setup: Kernel has A, B, C, D
+ await k.run(
+ [
+ ExecutionRequest(cell_id="0", code="a = 1"),
+ ExecutionRequest(cell_id="1", code="b = 2"),
+ ExecutionRequest(cell_id="2", code="c = 3"),
+ ExecutionRequest(cell_id="3", code="d = 4"),
+ ]
+ )
+ assert len(k.graph.cells) == 4
+
+ # Action: File manager knows A and C, explicitly deletes B
+ # D is orphaned (not in file), B is explicitly deleted
+ await k.sync_graph(
+ cells={"0": "a = 1", "2": "c = 3"},
+ run_ids=[],
+ delete_ids=["1"],
+ )
+
+ # Verify: Both B and D are deleted, only A and C remain
+ assert len(k.graph.cells) == 2
+ assert "0" in k.graph.cells
+ assert "2" in k.graph.cells
+ assert "1" not in k.graph.cells
+ assert "3" not in k.graph.cells
+ assert "a" in k.globals
+ assert "b" not in k.globals
+ assert "c" in k.globals
+ assert "d" not in k.globals
+
+ async def test_sync_graph_uninstantiated_request_cleanup(
+ self, any_kernel: Kernel
+ ) -> None:
+ """Test cleanup of uninstantiated requests when cells are deleted."""
+ k = any_kernel
+ # Setup: Instantiate with autorun=False, then run some cells
+ await k.instantiate(
+ CreationRequest(
+ execution_requests=(
+ ExecutionRequest(cell_id="0", code="x = 0"),
+ ExecutionRequest(cell_id="1", code="y = 1"),
+ ExecutionRequest(cell_id="2", code="z = 2"),
+ ),
+ set_ui_element_value_request=SetUIElementValueRequest.from_ids_and_values(
+ []
+ ),
+ auto_run=False,
+ )
+ )
+ assert len(k._uninstantiated_execution_requests) == 3
+
+ # Run cells 0 and 1 to add them to graph, keeping cell 2 uninstantiated
+ await k.run(
+ [
+ ExecutionRequest(cell_id="0", code="x = 0"),
+ ExecutionRequest(cell_id="1", code="y = 1"),
+ ]
+ )
+ # Cell 2 is still uninstantiated, cells 0 and 1 are in graph
+ assert len(k.graph.cells) == 2
+ assert "2" in k._uninstantiated_execution_requests
+
+ # Action: Sync with only cell 0, cell 1 is orphaned (deleted from file)
+ await k.sync_graph(
+ cells={"0": "x = 0"},
+ run_ids=[],
+ delete_ids=[],
+ )
+
+ # Verify: Uninstantiated request and graph entry for cell 1 are cleaned up
+ assert len(k.graph.cells) == 1
+ assert "0" in k.graph.cells
+ assert "1" not in k.graph.cells
+ # Cell 1's uninstantiated request should be cleaned up if it existed
+ assert "1" not in k._uninstantiated_execution_requests
+ # Cell 2 still has uninstantiated request (it was never in the graph)
+ assert "2" in k._uninstantiated_execution_requests
+
+ async def test_sync_graph_run_and_delete_combined(
+ self, any_kernel: Kernel
+ ) -> None:
+ """Test simultaneous run and delete in single sync operation."""
+ k = any_kernel
+ # Setup: Create A→B→C dependency chain
+ await k.run(
+ [
+ ExecutionRequest(cell_id="0", code="x = 1"),
+ ExecutionRequest(cell_id="1", code="y = x + 1"),
+ ExecutionRequest(cell_id="2", code="z = y + 1"),
+ ]
+ )
+ assert k.globals["z"] == 3
+ assert len(k.graph.cells) == 3
+
+ # Action: Delete B, update A and C (C no longer depends on B)
+ await k.sync_graph(
+ cells={"0": "x = 10", "2": "z = x + 5"},
+ run_ids=["0", "2"],
+ delete_ids=["1"],
+ )
+
+ # Verify: B deleted, A and C updated
+ assert len(k.graph.cells) == 2
+ assert "0" in k.graph.cells
+ assert "2" in k.graph.cells
+ assert "1" not in k.graph.cells
+ assert k.globals["x"] == 10
+ assert "y" not in k.globals
+ assert k.globals["z"] == 15
+
+ async def test_sync_graph_error_propagation(
+ self, any_kernel: Kernel
+ ) -> None:
+ """Test error propagation when sync introduces errors."""
+ k = any_kernel
+ # Setup: Cell 0 defines x
+ await k.run(
+ [
+ ExecutionRequest(cell_id="0", code="x = 1"),
+ ]
+ )
+ assert "x" in k.globals
+ assert not k.errors
+
+ # Action: Sync with new cell 1 that also defines x (multiple definition)
+ # Running both cells to trigger the error in both
+ await k.sync_graph(
+ cells={"0": "x = 1", "1": "x = 2"},
+ run_ids=["0", "1"],
+ delete_ids=[],
+ )
+
+ # Verify: Multiple definition error in both cells
+ assert "x" not in k.globals
+ assert set(k.errors.keys()) == {"0", "1"}
+ assert isinstance(k.errors["0"][0], MultipleDefinitionError)
+ assert isinstance(k.errors["1"][0], MultipleDefinitionError)
+
+ async def test_sync_graph_empty_sync(self, any_kernel: Kernel) -> None:
+ """Test no-op sync when already in sync."""
+ k = any_kernel
+ # Setup: Create cells
+ await k.run(
+ [
+ ExecutionRequest(cell_id="0", code="x = 1"),
+ ExecutionRequest(cell_id="1", code="y = 2"),
+ ]
+ )
+ assert k.globals["x"] == 1
+ assert k.globals["y"] == 2
+
+ # Action: Sync with same state (no changes)
+ await k.sync_graph(
+ cells={"0": "x = 1", "1": "y = 2"},
+ run_ids=[],
+ delete_ids=[],
+ )
+
+ # Verify: Nothing changed
+ assert len(k.graph.cells) == 2
+ assert k.globals["x"] == 1
+ assert k.globals["y"] == 2
+ assert not k.errors
+
class TestStrictExecution:
@staticmethod
diff --git a/tests/_server/test_sessions.py b/tests/_server/test_sessions.py
index 93b1a24b44..1d56f82fde 100644
--- a/tests/_server/test_sessions.py
+++ b/tests/_server/test_sessions.py
@@ -26,9 +26,9 @@ from marimo._messaging.ops import UpdateCellCodes
from marimo._runtime.requests import (
AppMetadata,
CreationRequest,
- ExecuteMultipleRequest,
ExecutionRequest,
SetUIElementValueRequest,
+ SyncGraphRequest,
)
from marimo._server.file_manager import AppFileManager
from marimo._server.file_router import AppFileRouter
@@ -762,7 +762,7 @@ def __():
# Verify that cells were queued for execution
assert session.session_view.add_control_request.called
last_call = session.session_view.add_control_request.call_args[0][0]
- assert isinstance(last_call, ExecuteMultipleRequest)
+ assert isinstance(last_call, SyncGraphRequest)
# Now change config to lazy mode
config_reader_lazy = config_reader.with_overrides(