Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

refactor: split up DirectedGraph into smaller components (#7527)

Myles Scolnick004c787a7ea2
diff --git a/marimo/_ast/toplevel.py b/marimo/_ast/toplevel.py
index c3bc5a4a5b..228327809c 100644
--- a/marimo/_ast/toplevel.py
+++ b/marimo/_ast/toplevel.py
@@ -17,7 +17,8 @@ from marimo._ast.names import (
)
from marimo._ast.variables import BUILTINS
from marimo._ast.visitor import Name, VariableData
-from marimo._runtime.dataflow import DirectedGraph, topological_sort
+from marimo._runtime.dataflow import topological_sort
+from marimo._runtime.dataflow.topology import GraphTopology
from marimo._types.ids import CellId_t
if TYPE_CHECKING:
@@ -379,7 +380,7 @@ class TopLevelExtraction:
@classmethod
def from_graph(
cls,
- graph: DirectedGraph,
+ graph: GraphTopology,
cell: CellImpl | None = None,
) -> TopLevelExtraction:
if cell:
diff --git a/marimo/_lint/validate_graph.py b/marimo/_lint/validate_graph.py
index 992526ea77..8e798a0b2b 100644
--- a/marimo/_lint/validate_graph.py
+++ b/marimo/_lint/validate_graph.py
@@ -12,6 +12,7 @@ from marimo._messaging.errors import (
SetupRootError,
)
from marimo._runtime.dataflow import DirectedGraph
+from marimo._runtime.dataflow.topology import GraphTopology
from marimo._types.ids import CellId_t
@@ -68,7 +69,7 @@ def check_for_multiple_definitions(
def check_for_invalid_root(
- graph: DirectedGraph,
+ graph: GraphTopology,
) -> dict[CellId_t, list[SetupRootError]]:
"""Setup cell cannot have parents."""
errors: dict[CellId_t, list[SetupRootError]] = defaultdict(list)
diff --git a/marimo/_runtime/dataflow.py b/marimo/_runtime/dataflow.py
deleted file mode 100644
index 4148c7a24b..0000000000
--- a/marimo/_runtime/dataflow.py
+++ /dev/null
@@ -1,1043 +0,0 @@
-# Copyright 2024 Marimo. All rights reserved.
-from __future__ import annotations
-
-import threading
-from collections import deque
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, cast
-
-from marimo import _loggers
-from marimo._ast.cell import CellImpl
-from marimo._ast.compiler import code_key
-from marimo._ast.sql_visitor import SQLRef, SQLTypes
-from marimo._ast.variables import is_mangled_local
-from marimo._ast.visitor import ImportData, Name, VariableData
-from marimo._runtime.executor import (
- ExecutionConfig,
- get_executor,
-)
-from marimo._types.ids import CellId_t
-
-if TYPE_CHECKING:
- from collections.abc import Collection
-
-
-Edge = tuple[CellId_t, CellId_t]
-# EdgeWithVar uses a list rather than a set for the variables linking the cells
-# as sets are not JSON-serializable (required by static_notebook_template()).
-# The first entry is the source node; the second entry is a list of defs from
-# the source read by the destination; and the third entry is the destination
-# node.
-EdgeWithVar = tuple[CellId_t, tuple[str, ...], CellId_t]
-
-LOGGER = _loggers.marimo_logger()
-
-
-# TODO(akshayka): Add method disable_cell, enable_cell which handle
-# state transitions on cells
-@dataclass(frozen=True)
-class DirectedGraph:
- # Nodes in the graph
- cells: dict[CellId_t, CellImpl] = field(default_factory=dict)
-
- # Edge (u, v) means v is a child of u, i.e., v has a reference
- # to something defined in u
- children: dict[CellId_t, set[CellId_t]] = field(default_factory=dict)
-
- # Reversed edges (parent pointers) for convenience
- parents: dict[CellId_t, set[CellId_t]] = field(default_factory=dict)
-
- # Cells that define the same name
- #
- # siblings[cell_id] is a set of cell ids, one for each cell that shares a
- # definition with cell_id.
- #
- # If this dict is non-empty, then the marimo program contains multiply
- # defined names (and is therefore in an error state)
- siblings: dict[CellId_t, set[CellId_t]] = field(default_factory=dict)
-
- # A mapping from defs to the cells that define them
- definitions: dict[Name, set[CellId_t]] = field(default_factory=dict)
- typed_definitions: dict[tuple[Name, str], set[CellId_t]] = field(
- default_factory=dict
- )
- definition_types: dict[Name, set[str]] = field(default_factory=dict)
-
- # The set of cycles in the graph
- cycles: set[tuple[Edge, ...]] = field(default_factory=set)
-
- # This lock must be acquired during methods that mutate the graph; it's
- # only needed because a graph is shared between the kernel and the code
- # completion service. It should almost always be uncontended.
- lock: threading.Lock = field(default_factory=threading.Lock)
-
- def is_cell_cached(self, cell_id: CellId_t, code: str) -> bool:
- """Whether a cell with id `cell_id` and code `code` is in the graph."""
- return (
- cell_id in self.cells and code_key(code) == self.cells[cell_id].key
- )
-
- # TODO: language type?
- def get_defining_cells(self, name: Name) -> set[CellId_t]:
- """Get all cells that define name.
-
- This is a singleton for well-formed graphs.
- """
- return self.definitions[name]
-
- def get_referring_cells(
- self, name: Name, language: Literal["python", "sql"]
- ) -> set[CellId_t]:
- """Get all cells that have a ref to `name`.
-
- The variable can be either a Python variable or a SQL variable (table).
- SQL variables don't leak to Python cells, but Python variables do leak
- to SQL.
-
- Only does a local analysis of refs, without taking into consideration
- whether refs are defined by other cells.
- """
- if language == "sql":
- # For SQL, only return SQL cells that reference the name
- cells = set()
- for cid, cell in self.cells.items():
- if cell.language != "sql":
- continue
-
- for ref in cell.refs:
- # Direct reference match
- if ref == name:
- cells.add(cid)
- break
-
- sql_ref = cell.sql_refs.get(ref)
-
- kind: SQLTypes = "any"
- if name in cell.variable_data:
- variable_data = cell.variable_data[name][-1]
- kind = cast(SQLTypes, variable_data.kind)
-
- # Hierarchical reference match
- if sql_ref and sql_ref.matches_hierarchical_ref(
- name, ref, kind
- ):
- cells.add(cid)
- break
-
- return cells
- else:
- # For Python, return all cells that reference the name
- return {
- cid for cid, cell in self.cells.items() if name in cell.refs
- }
-
- def _find_sql_hierarchical_matches(
- self, sql_ref: SQLRef
- ) -> list[tuple[set[CellId_t], Name]]:
- """
- This method searches through all definitions in the graph to find cells
- that define the individual components (table, schema, or catalog) of the
- hierarchical reference.
-
- For example, given a reference "my_schema.my_table", this method will:
- - Look for cells that define a table/view named "my_table"
- - Look for cells that define a catalog named "my_schema"
- (when the reference has at least 2 parts)
-
- Args:
- sql_ref: A hierarchical SQL reference (e.g., "schema.table",
- "catalog.schema.table") to find matching definitions for.
-
- Returns:
- A list of tuples containing:
- - A set of cell IDs that define components of the hierarchical reference
- - The definition of the name that was found (e.g., "schema.table" -> "table")
- """
- matching_cell_ids_list = []
-
- for (def_name, kind), cell_ids in self.typed_definitions.items():
- # Match table/view definitions
- if sql_ref.contains_hierarchical_ref(def_name, kind):
- matching_cell_ids_list.append((cell_ids, def_name))
-
- return matching_cell_ids_list
-
- def _is_valid_cell_reference(
- self, cell_id: CellId_t, variable_name: Name
- ) -> bool:
- """Check if a cell reference is valid and log errors if not."""
- if cell_id not in self.cells:
- LOGGER.error(
- "Variable %s is defined in cell %s, but is not in the graph",
- variable_name,
- cell_id,
- )
- return False
- return True
-
- def _resolve_variable_name(
- self,
- name: Name,
- other_cell: CellImpl,
- sql_ref: Optional[SQLRef],
- sql_matches: list[tuple[set[CellId_t], Name]],
- ) -> Name:
- """
- Resolve the variable name to use when checking if it exists in another cell.
-
- For regular (non-SQL) references, returns the original name unchanged.
- For SQL hierarchical references, finds the variable name from sql_matches that
- is actually defined in the other_cell.
-
- Example:
- cell_1: CREATE SCHEMA schema_name
- cell_2: CREATE TABLE schema_name.table_name
- cell_3: FROM schema_name.table_name SELECT *
-
- When cell_3 references "schema_name.table_name":
- - For cell_1: returns "schema_name" (what cell_1 actually defines)
- - For cell_2: returns "table_name" (what cell_2 actually defines)
- """
- if not sql_ref or name in other_cell.variable_data:
- return name
-
- # For SQL hierarchical references, find the matching variable name
- for _, matching_variable_name in sql_matches:
- if matching_variable_name in other_cell.variable_data:
- return matching_variable_name
-
- return name
-
- def get_path(self, source: CellId_t, dst: CellId_t) -> list[Edge]:
- """Get a path from `source` to `dst`, if any."""
- if source == dst:
- return []
-
- # deque has O(1) append/pop operation
- queue: deque[tuple[CellId_t, list[Edge]]] = deque([(source, [])])
- found = {source} # set has O(1) lookups
-
- while queue:
- node, path = queue.popleft() # O(1) operation
- for cid in self.children[node]:
- if cid not in found:
- next_path = path + [(node, cid)]
- if cid == dst:
- return next_path
- found.add(cid)
- queue.append((cid, next_path))
- return []
-
- def register_cell(self, cell_id: CellId_t, cell: CellImpl) -> None:
- """Add a cell to the graph.
-
- Mutates the graph, acquiring `self.lock`.
-
- Requires that `cell_id` is not already in the graph.
- """
- LOGGER.debug("Acquiring graph lock to register cell %s", cell_id)
- with self.lock:
- LOGGER.debug("Acquired graph lock.")
- assert cell_id not in self.cells
- self.cells[cell_id] = cell
- # Children are the set of cells that refer to a name defined in
- # `cell`
- children: set[CellId_t] = set()
- # Cells that define the same name as this one
- siblings: set[CellId_t] = set()
- # Parents are the set of cells that define a name referred to by
- # `cell`
- parents: set[CellId_t] = set()
-
- # Populate children, siblings, and parents
- self.children[cell_id] = children
- self.siblings[cell_id] = siblings
- self.parents[cell_id] = parents
-
- # First, we process the variables that this cell defines. Any cell
- # that refers to a defined variable becomes a child of this cell;
- # any cell that defines a variable defined by this cell becomes
- # a sibling.
- for name, variable_data in cell.variable_data.items():
- # NB. Only the last definition matters.
- # Technically more nuanced with branching statements, but this is
- # the best we can do with static analysis.
- variable = variable_data[-1]
- typed_def = (name, variable.kind)
-
- if (
- name in self.definitions
- and typed_def not in self.typed_definitions
- ):
- # Duplicate if the qualified name is no different
- if (
- variable.qualified_name == name
- or variable.language != "sql"
- ):
- self.definitions[name].add(cell_id)
- else:
- self.definitions.setdefault(name, set()).add(cell_id)
- self.typed_definitions.setdefault(typed_def, set()).add(
- cell_id
- )
- self.definition_types.setdefault(name, set()).add(
- variable.kind
- )
-
- for sibling in self.definitions[name]:
- # TODO(akshayka): Distinguish between Python/SQL?
- if sibling != cell_id:
- siblings.add(sibling)
- self.siblings[sibling].add(cell_id)
-
- # a cell can refer to its own defs, but that doesn't add an
- # edge to the dependency graph
- referring_cells = self.get_referring_cells(
- name,
- language=variable_data[-1].language,
- ) - set((cell_id,))
- # we will add an edge (cell_id, v) for each v in
- # referring_cells; if there is a path from v to cell_id, then
- # the new edge will form a cycle
- for v in referring_cells:
- path = self.get_path(v, cell_id)
- if path:
- self.cycles.add(tuple([(cell_id, v)] + path))
-
- children.update(referring_cells)
- for child in referring_cells:
- self.parents[child].add(cell_id)
-
- # Next, we process the cells references. The cell becomes a child
- # of cells that define its referenced variables. We also have
- # special logic for handling references that are deleted by this cell,
- # since cells that delete variables that were defined elsewhere
- # are made children of cells that reference that variable.
-
- for name in cell.refs:
- # First, for each referenced variable, we add cells that define
- # that variable as parents
- other_ids_defining_name: set[CellId_t] = (
- self.definitions[name]
- if name in self.definitions
- else set()
- ) - set((cell_id,))
-
- variable_name: Name = name
-
- # Handle SQL matching for hierarchical references
- sql_matches: list[tuple[set[CellId_t], Name]] = []
- sql_ref = cell.sql_refs.get(name)
- if sql_ref:
- sql_matches = self._find_sql_hierarchical_matches(sql_ref)
- for matching_cell_ids, _ in sql_matches:
- if cell_id in matching_cell_ids:
- LOGGER.debug(
- "Cell %s is referencing itself", cell_id
- )
- continue
- other_ids_defining_name.update(matching_cell_ids)
-
- # If other_ids_defining_name is empty, the user will get a
- # NameError at runtime (unless the symbol is a builtin).
- for other_id in other_ids_defining_name:
- if other_id == cell_id:
- LOGGER.error("Cell %s is referencing itself", cell_id)
- continue
- if not self._is_valid_cell_reference(
- other_id, variable_name
- ):
- continue
- other_cell = self.cells[other_id]
-
- variable_name = self._resolve_variable_name(
- variable_name, other_cell, sql_ref, sql_matches
- )
-
- # If we don't have a matching variable name, skip
- if variable_name not in other_cell.variable_data:
- LOGGER.error(
- "Variable %s is not defined in cell %s",
- variable_name,
- other_id,
- )
- continue
-
- other_variable_data = other_cell.variable_data[
- variable_name
- ][-1]
- language = other_variable_data.language
- if language == "sql" and cell.language == "python":
- # SQL table/db def -> Python ref is not an edge
- continue
- if language == "sql" and cell.language == "sql":
- # Edges between SQL cells need to respect hierarchy.
- if sql_ref and not sql_ref.matches_hierarchical_ref(
- variable_name,
- other_variable_data.qualified_name or name,
- kind=cast(SQLTypes, other_variable_data.kind),
- ):
- continue
- parents.add(other_id)
- # we are adding an edge (other_id, cell_id). If there
- # is a path from cell_id to other_id, then the new
- # edge forms a cycle
- path = self.get_path(cell_id, other_id)
- if path:
- self.cycles.add(tuple([(other_id, cell_id)] + path))
- self.children[other_id].add(cell_id)
-
- # Next, any cell that deletes this referenced variable is made
- # a child of this cell. In particular, if a cell deletes a
- # variable, it becomes a child of all other cells that
- # reference that variable. This means that if two cells delete
- # the same variable, they form a cycle.
- #
- # For example, two cells
- #
- # cell u: x
- # cell v: del x
- #
- # v becomes a child of u.
- #
- # Another example:
- #
- # cell u: del x
- # cell v: del x
- #
- # u and v form a cycle.
- other_ids_deleting_name: set[CellId_t] = set(
- cid
- for cid in self.get_referring_cells(
- name, language="python"
- )
- if name in self.cells[cid].deleted_refs
- ) - set((cell_id,))
- for v in other_ids_deleting_name:
- path = self.get_path(v, cell_id)
- if path:
- self.cycles.add(tuple([(cell_id, v)] + path))
- self.parents[v].add(cell_id)
- children.update(other_ids_deleting_name)
-
- # Finally, if this cell deletes a variable, we make it a child of
- # all other cells that reference this variable.
- for name in cell.deleted_refs:
- referring_cells = self.get_referring_cells(
- name, language="python"
- ) - set((cell_id,))
- for other_id in referring_cells:
- parents.add(other_id)
- path = self.get_path(cell_id, other_id)
- if path:
- self.cycles.add(tuple([(other_id, cell_id)] + path))
- self.children[other_id].add(cell_id)
-
- LOGGER.debug("Registered cell %s and released graph lock", cell_id)
- if self.is_any_ancestor_stale(cell_id):
- self.set_stale(set([cell_id]))
-
- if self.is_any_ancestor_disabled(cell_id):
- cell.set_runtime_state(status="disabled-transitively")
-
- def is_any_ancestor_stale(self, cell_id: CellId_t) -> bool:
- return any(self.cells[cid].stale for cid in self.ancestors(cell_id))
-
- def is_any_ancestor_disabled(self, cell_id: CellId_t) -> bool:
- return any(
- self.cells[cid].config.disabled for cid in self.ancestors(cell_id)
- )
-
- def disable_cell(self, cell_id: CellId_t) -> None:
- """
- Disables a cell in the graph.
-
- Does not mutate the graph (but does mutate cell statuses).
-
- Returns the ids of descendants that are disabled transitively.
- """
- if cell_id not in self.cells:
- raise ValueError(f"Cell {cell_id} not found")
-
- for cid in transitive_closure(self, set([cell_id])) - set([cell_id]):
- cell = self.cells[cid]
- cell.set_runtime_state(status="disabled-transitively")
-
- def enable_cell(self, cell_id: CellId_t) -> set[CellId_t]:
- """
- Enables a cell in the graph.
-
- Does not mutate the graph (but does mutate cell statuses).
-
- Returns:
- - set of cells that were stale and should be re-run
- """
- if cell_id not in self.cells:
- raise ValueError(f"Cell {cell_id} not found")
-
- cells_to_run: set[CellId_t] = set()
- for cid in transitive_closure(self, set([cell_id])):
- if not self.is_disabled(cid):
- child = self.cells[cid]
- if child.stale:
- # cell was previously disabled, is no longer
- # disabled, and is stale: needs to run.
- cells_to_run.add(cid)
- if child.disabled_transitively:
- # cell is no longer disabled: status -> idle
- child.set_runtime_state("idle")
- return cells_to_run
-
- def delete_cell(self, cell_id: CellId_t) -> set[CellId_t]:
- """Removes a cell from the graph.
-
- Mutates the graph, acquiring `self.lock`.
-
- Returns the ids of the children of the removed cell.
- """
- LOGGER.debug("Acquiring graph lock to delete cell %s", cell_id)
- with self.lock:
- LOGGER.debug("Acquired graph lock to delete cell %s", cell_id)
- if cell_id not in self.cells:
- raise ValueError(f"Cell {cell_id} not found")
-
- # Removing this cell from its defs' definer sets
- for name in self.cells[cell_id].defs:
- name_defs = self.definitions[name]
- name_defs.remove(cell_id)
- if not name_defs:
- # No more cells define this name, so we remove it from the
- # graph
- del self.definitions[name]
- # Clean up all typed definitions
- for typed_def in self.definition_types[name]:
- del self.typed_definitions[(name, typed_def)]
- del self.definition_types[name]
-
- # Remove cycles that are broken from removing this cell.
- edges = [(cell_id, child) for child in self.children[cell_id]] + [
- (parent, cell_id) for parent in self.parents[cell_id]
- ]
- for e in edges:
- broken_cycles = [c for c in self.cycles if e in c]
- for c in broken_cycles:
- self.cycles.remove(c)
-
- # Grab a reference to children before we remove it from our map.
- children = self.children[cell_id]
-
- # Purge this cell from the graph.
- del self.cells[cell_id]
- del self.children[cell_id]
- del self.parents[cell_id]
- del self.siblings[cell_id]
-
- for elems in self.parents.values():
- if cell_id in elems:
- elems.remove(cell_id)
- for elems in self.children.values():
- if cell_id in elems:
- elems.remove(cell_id)
- for elems in self.siblings.values():
- if cell_id in elems:
- elems.remove(cell_id)
- LOGGER.debug("Deleted cell %s and Released graph lock.", cell_id)
- return children
-
- def is_disabled(self, cell_id: CellId_t) -> bool:
- if cell_id not in self.cells:
- raise ValueError(f"Cell {cell_id} not in graph.")
- cell = self.cells[cell_id]
- if cell.config.disabled:
- return True
- seen: set[CellId_t] = set()
- queue = [cell_id]
- while queue:
- cid = queue.pop()
- seen.add(cid)
- for parent_id in self.parents[cid]:
- if parent_id in seen:
- continue
- elif self.cells[parent_id].config.disabled:
- return True
- else:
- queue.append(parent_id)
- return False
-
- def get_imports(
- self, cell_id: Optional[CellId_t] = None
- ) -> dict[Name, ImportData]:
- imports = {}
- cells = (
- self.cells.values() if cell_id is None else [self.cells[cell_id]]
- )
- for cell in cells:
- for imported in cell.imports:
- imports[imported.definition] = imported
- return imports
-
- def get_multiply_defined(self) -> list[Name]:
- """Return a list of names that are defined in multiple cells"""
- names: list[Name] = []
- for name, definers in self.definitions.items():
- if len(definers) > 1:
- names.append(name)
- return names
-
- def get_deleted_nonlocal_ref(self) -> list[Name]:
- names: list[Name] = []
- for cell in self.cells.values():
- for ref in cell.deleted_refs:
- if ref in self.definitions:
- names.append(ref)
- return names
-
- def descendants(self, cell_id: CellId_t) -> set[CellId_t]:
- return transitive_closure(self, set([cell_id]), inclusive=False)
-
- def ancestors(self, cell_id: CellId_t) -> set[CellId_t]:
- return transitive_closure(
- self, set([cell_id]), children=False, inclusive=False
- )
-
- def set_stale(
- self, cell_ids: set[CellId_t], prune_imports: bool = False
- ) -> None:
- relatives = None if not prune_imports else import_block_relatives
- for cid in transitive_closure(self, cell_ids, relatives=relatives):
- self.cells[cid].set_stale(stale=True)
-
- def get_stale(self) -> set[CellId_t]:
- return set([cid for cid, cell in self.cells.items() if cell.stale])
-
- def get_transitive_references(
- self,
- refs: set[Name],
- inclusive: bool = True,
- predicate: Callable[[Name, VariableData], bool] | None = None,
- ) -> set[Name]:
- """Return a set of the passed-in cells' references and their
- references on the block (function / class) level.
-
- If inclusive, includes the references of the passed-in cells in the
- set.
-
- If predicate, only references satisfying predicate(ref) are included
- """
- # TODO: Consider caching on the graph level and updating on register /
- # delete
- processed: set[Name] = set()
- queue: set[Name] = refs & self.definitions.keys()
- predicate = predicate or (lambda *_: True)
-
- while queue:
- # Should ideally be one cell per ref, but for completion, stay
- # agnostic to potenital cycles.
- cells = {
- cell_id
- for ref in queue
- for cell_id in self.definitions.get(ref, set())
- }
-
- for cell_id in cells:
- data = self.cells[cell_id].variable_data
- newly_processed = set(data.keys()) & queue
- processed.update(newly_processed)
- queue.difference_update(newly_processed)
- for variable in newly_processed:
- # variables can be defined multiple times in a single
- # cell ...
- for datum in data[variable]:
- if predicate(variable, datum):
- to_process = datum.required_refs - processed
- queue.update(to_process & self.definitions.keys())
- # Private variables referenced by public functions
- # have to be included.
- processed.update(
- maybe_private
- for maybe_private in (
- to_process - self.definitions.keys()
- )
- if is_mangled_local(maybe_private, cell_id)
- )
-
- if inclusive:
- return processed | refs
- return processed - refs
-
- def copy(self, filename: None | str = None) -> DirectedGraph:
- """Return a deep copy of the graph by recompiling all cells.
-
- This is mainly useful in the case where recompilation must be done
- due to a dynamically changing notebook, where the line cache must be
- consistent with the cell code, e.g. for debugging.
- """
- from marimo._ast.compiler import compile_cell
-
- graph = DirectedGraph()
- with self.lock:
- for cid, old_cell in self.cells.items():
- cell = compile_cell(
- old_cell.code,
- cell_id=cid,
- filename=filename,
- )
- # Carry over import data manually
- imported_defs = old_cell.import_workspace.imported_defs
- is_import_block = old_cell.import_workspace.is_import_block
- cell.import_workspace.imported_defs = imported_defs
- cell.import_workspace.is_import_block = is_import_block
- # Reregister
- graph.register_cell(cid, cell)
- return graph
-
-
-def transitive_closure(
- graph: DirectedGraph,
- cell_ids: set[CellId_t],
- children: bool = True,
- inclusive: bool = True,
- relatives: (
- Callable[[DirectedGraph, CellId_t, bool], set[CellId_t]] | None
- ) = None,
- predicate: Callable[[CellImpl], bool] | None = None,
-) -> set[CellId_t]:
- """Return a set of the passed-in cells and their descendants or ancestors
-
- If children is True, returns descendants; otherwise, returns ancestors
-
- If inclusive, includes passed-in cells in the set.
-
- If relatives is not None, it computes the parents/children of a
- cell
-
- If predicate, only cells satisfying predicate(cell) are included; applied
- after the relatives are computed
- """
-
- result: set[CellId_t] = cell_ids.copy() if inclusive else set()
- seen: set[CellId_t] = cell_ids.copy()
- queue: deque[CellId_t] = deque(cell_ids)
- predicate = predicate or (lambda _: True)
-
- def _relatives(cid: CellId_t) -> set[CellId_t]:
- if relatives is None:
- return graph.children[cid] if children else graph.parents[cid]
- return relatives(graph, cid, children)
-
- while queue:
- cid = queue.popleft() # O(1) operation
-
- relatives_set = _relatives(cid)
- new_relatives = relatives_set - seen
-
- if new_relatives:
- # Add new relatives to queue and result if they pass predicate
- for relative in new_relatives:
- if predicate(graph.cells[relative]):
- result.add(relative)
- seen.add(relative)
- queue.append(relative)
-
- return result
-
-
-def induced_subgraph(
- graph: DirectedGraph, cell_ids: Collection[CellId_t]
-) -> tuple[dict[CellId_t, set[CellId_t]], dict[CellId_t, set[CellId_t]]]:
- """Return parents and children for each node in `cell_ids`
-
- Represents the subgraph induced by `cell_ids`.
- """
- parents: dict[CellId_t, set[CellId_t]] = {}
- children: dict[CellId_t, set[CellId_t]] = {}
- for cid in cell_ids:
- parents[cid] = set(p for p in graph.parents[cid] if p in cell_ids)
- children[cid] = set(c for c in graph.children[cid] if c in cell_ids)
- return parents, children
-
-
-def get_cycles(
- graph: DirectedGraph, cell_ids: Collection[CellId_t]
-) -> list[tuple[Edge, ...]]:
- """Get all cycles among `cell_ids`."""
- _, induced_children = induced_subgraph(graph, cell_ids)
- induced_edges = set(
- [(u, v) for u in induced_children for v in induced_children[u]]
- )
- return [c for c in graph.cycles if all(e in induced_edges for e in c)]
-
-
-def topological_sort(
- graph: DirectedGraph, cell_ids: Collection[CellId_t]
-) -> list[CellId_t]:
- """Sort `cell_ids` in a topological order using a heap queue.
-
- When multiple cells have the same parents (including no parents), the tie is broken by
- registration order - cells registered earlier are processed first.
- """
- from heapq import heapify, heappop, heappush
-
- # Use a list for O(1) lookup of registration order
- registration_order = list(graph.cells.keys())
- top_down_keys = {key: idx for idx, key in enumerate(registration_order)}
-
- # Build adjacency lists and in-degree counts
- parents, children = induced_subgraph(graph, cell_ids)
- in_degree = {cid: len(parents[cid]) for cid in cell_ids}
-
- # Initialize heap with roots
- heap = [
- (top_down_keys[cid], cid) for cid in cell_ids if in_degree[cid] == 0
- ]
- heapify(heap)
-
- sorted_cell_ids: list[CellId_t] = []
- while heap:
- _, cid = heappop(heap)
- sorted_cell_ids.append(cid)
-
- # Process children
- for child in children[cid]:
- in_degree[child] -= 1
- if in_degree[child] == 0:
- heappush(heap, (top_down_keys[child], child))
-
- return sorted_cell_ids
-
-
-def prune_cells_for_overrides(
- graph: DirectedGraph,
- execution_order: Collection[CellId_t],
- overrides: dict[str, Any],
-) -> list[CellId_t]:
- """Prune cells from execution when their definitions are overridden.
-
- When variable definitions are provided externally (overrides), this function
- identifies cells that would normally define those variables and removes them
- from the execution order. It also validates that all definitions from pruned
- cells are provided in the overrides.
-
- Args:
- graph: The dataflow graph containing cell dependencies
- execution_order: Ordered collection of cells to execute
- overrides: Dictionary mapping variable names to their override values
-
- Returns:
- Filtered execution order excluding cells whose definitions are overridden
-
- Raises:
- IncompleteRefsError: If overrides don't provide all definitions from
- pruned cells (e.g., a cell defines both x and y, but only x is
- provided in overrides)
-
- Example:
- If cell A defines variables x and y, and overrides = {"x": 1, "y": 2},
- then cell A will be pruned from execution_order. However, if overrides
- only contains {"x": 1}, an IncompleteRefsError is raised because y is
- missing.
- """
- if not overrides:
- return list(execution_order)
-
- cells_to_prune: set[CellId_t] = set()
- refs = set(overrides.keys())
-
- # Find cells that define the overridden variables
- for ref_name in refs:
- if ref_name in graph.definitions:
- defining_cells = graph.get_defining_cells(ref_name)
- cells_to_prune.update(defining_cells)
-
- # Validate that all definitions from pruned cells are provided
- missing_defs: set[str] = set()
- for cell_id in cells_to_prune:
- cell = graph.cells[cell_id]
- # Check all definitions this cell would have provided
- for missing in cell.defs - refs:
- missing_defs.add(missing)
-
- if missing_defs:
- from marimo._ast.errors import IncompleteRefsError
-
- raise IncompleteRefsError(
- f"When providing refs that override cell definitions, you must "
- f"provide all definitions from those cells. Missing: {sorted(missing_defs)}. "
- f"Provided refs: {sorted(refs)}."
- )
-
- # Return filtered execution order
- return [cid for cid in execution_order if cid not in cells_to_prune]
-
-
-def import_block_relatives(
- graph: DirectedGraph, cid: CellId_t, children: bool
-) -> set[CellId_t]:
- if not children:
- return graph.parents[cid]
-
- cell = graph.cells[cid]
- if not cell.import_workspace.is_import_block:
- return graph.children[cid]
-
- # This cell is an import block, which should be special cased:
- #
- # We prune definitions that have already been imported from the set of
- # definitions used to find the descendants of this cell.
- unimported_defs = cell.defs - cell.import_workspace.imported_defs
-
- children_ids = {
- child_id
- for name in unimported_defs
- for child_id in graph.get_referring_cells(name, language="python")
- }
-
- # If children haven't been executed, then still use imported defs;
- # handle an edge case when an import cell is interrupted by an
- # exception or user interrupt, so that a module is imported but the
- # cell's children haven't run.
- if cell.import_workspace.imported_defs:
- interrupted_states = {"interrupted", "cancelled", "marimo-error", None}
- children_ids.update(
- child_id
- for name in cell.import_workspace.imported_defs
- for child_id in graph.get_referring_cells(name, language="python")
- if graph.cells[child_id].run_result_status in interrupted_states
- )
-
- return children_ids
-
-
-class Runner:
- """Utility for running individual cells in a graph
-
- This class provides methods to a run a cell in the graph and obtain its
- output (last expression) and the values of its defs.
-
- If needed, the runner will recursively compute the values of the cell's
- refs by executing its ancestors. Refs can also be substituted by the
- caller.
-
- TODO(akshayka): Add an API for caching defs across cell runs.
- """
-
- def __init__(self, graph: DirectedGraph) -> None:
- self._graph = graph
- self._executor = get_executor(ExecutionConfig())
-
- @staticmethod
- def _returns(cell_impl: CellImpl, glbls: dict[str, Any]) -> dict[str, Any]:
- return {name: glbls[name] for name in cell_impl.defs if name in glbls}
-
- @staticmethod
- def _substitute_refs(
- cell_impl: CellImpl,
- glbls: dict[str, Any],
- kwargs: dict[str, Any],
- ) -> None:
- for argname, argvalue in kwargs.items():
- if argname in cell_impl.refs:
- glbls[argname] = argvalue
- else:
- raise ValueError(
- f"Cell got unexpected argument {argname}"
- f"The allowed arguments are {cell_impl.refs}."
- )
-
- def _get_ancestors(
- self, cell_impl: CellImpl, kwargs: dict[str, Any]
- ) -> set[CellId_t]:
- # Get the transitive closure of parents defining unsubstituted refs
- graph = self._graph
- substitutions = set(kwargs.keys())
- unsubstituted_refs = cell_impl.refs - substitutions
- parent_ids = set(
- [
- parent_id
- for parent_id in graph.parents[cell_impl.cell_id]
- if graph.cells[parent_id].defs.intersection(unsubstituted_refs)
- ]
- )
- return transitive_closure(graph, parent_ids, children=False)
-
- @staticmethod
- def _validate_kwargs(cell_impl: CellImpl, kwargs: dict[str, Any]) -> None:
- for argname in kwargs:
- if argname not in cell_impl.refs:
- raise ValueError(
- f"Cell got unexpected argument {argname}; "
- f"The allowed arguments are {cell_impl.refs}."
- )
-
- def is_coroutine(self, cell_id: CellId_t) -> bool:
- return self._graph.cells[cell_id].is_coroutine() or any(
- self._graph.cells[cid].is_coroutine()
- for cid in self._get_ancestors(
- self._graph.cells[cell_id], kwargs={}
- )
- )
-
- async def run_cell_async(
- self, cell_id: CellId_t, kwargs: dict[str, Any]
- ) -> tuple[Any, dict[str, Any]]:
- """Run a possibly async cell and its ancestors
-
- Substitutes kwargs as refs for the cell, omitting ancestors that
- whose refs are substituted.
- """
- graph = self._graph
- cell_impl = graph.cells[cell_id]
- Runner._validate_kwargs(cell_impl, kwargs)
- ancestor_ids = self._get_ancestors(cell_impl, kwargs)
-
- glbls: dict[str, Any] = {}
- for cid in topological_sort(graph, ancestor_ids):
- await self._executor.execute_cell_async(
- graph.cells[cid], glbls, graph
- )
-
- Runner._substitute_refs(cell_impl, glbls, kwargs)
- output = await self._executor.execute_cell_async(
- graph.cells[cell_impl.cell_id], glbls, graph
- )
- defs = Runner._returns(cell_impl, glbls)
- return output, defs
-
- def run_cell_sync(
- self, cell_id: CellId_t, kwargs: dict[str, Any]
- ) -> tuple[Any, dict[str, Any]]:
- """Run a synchronous cell and its ancestors
-
- Substitutes kwargs as refs for the cell, omitting ancestors that
- whose refs are substituted.
-
- Raises a `RuntimeError` if the cell or any of its unsubstituted
- ancestors are coroutine functions.
- """
- graph = self._graph
- cell_impl = graph.cells[cell_id]
- if cell_impl.is_coroutine():
- raise RuntimeError(
- "A coroutine function can't be run synchronously. "
- "Use `run_async()` instead"
- )
-
- Runner._validate_kwargs(cell_impl, kwargs)
- ancestor_ids = self._get_ancestors(cell_impl, kwargs)
-
- if any(graph.cells[cid].is_coroutine() for cid in ancestor_ids):
- raise RuntimeError(
- "Cell has an ancestor that is a "
- "coroutine (async) cell. Use `run_async()` instead"
- )
-
- glbls: dict[str, Any] = {}
- for cid in topological_sort(graph, ancestor_ids):
- self._executor.execute_cell(graph.cells[cid], glbls, graph)
-
- self._substitute_refs(cell_impl, glbls, kwargs)
- output = self._executor.execute_cell(
- graph.cells[cell_impl.cell_id], glbls, graph
- )
- defs = Runner._returns(cell_impl, glbls)
- return output, defs
diff --git a/marimo/_runtime/dataflow/__init__.py b/marimo/_runtime/dataflow/__init__.py
new file mode 100644
index 0000000000..94c4abd5c5
--- /dev/null
+++ b/marimo/_runtime/dataflow/__init__.py
@@ -0,0 +1,259 @@
+# Copyright 2024 Marimo. All rights reserved.
+from __future__ import annotations
+
+from collections import deque
+from typing import TYPE_CHECKING, Any, Callable
+
+from marimo import _loggers
+from marimo._ast.cell import CellImpl
+from marimo._runtime.dataflow.graph import DirectedGraph
+from marimo._runtime.dataflow.runner import Runner
+from marimo._runtime.dataflow.topology import GraphTopology
+from marimo._runtime.dataflow.types import Edge, EdgeWithVar
+from marimo._types.ids import CellId_t
+
+if TYPE_CHECKING:
+ from collections.abc import Collection
+
+
+LOGGER = _loggers.marimo_logger()
+
+
+def transitive_closure(
+ graph: GraphTopology,
+ cell_ids: set[CellId_t],
+ *,
+ children: bool = True,
+ inclusive: bool = True,
+ relatives: Callable[[CellId_t, bool], set[CellId_t]] | None = None,
+ predicate: Callable[[CellImpl], bool] | None = None,
+) -> set[CellId_t]:
+ """Return a set of the passed-in cells and their descendants or ancestors
+
+ If children is True, returns descendants; otherwise, returns ancestors
+
+ If inclusive, includes passed-in cells in the set.
+
+ If relatives is not None, it computes the parents/children of a
+ cell
+
+ If predicate, only cells satisfying predicate(cell) are included; applied
+ after the relatives are computed
+ """
+
+ result: set[CellId_t] = cell_ids.copy() if inclusive else set()
+ seen: set[CellId_t] = cell_ids.copy()
+ queue: deque[CellId_t] = deque(cell_ids)
+ predicate = predicate or (lambda _: True)
+
+ def _relatives(cid: CellId_t) -> set[CellId_t]:
+ if relatives is None:
+ return graph.children[cid] if children else graph.parents[cid]
+ return relatives(cid, children)
+
+ while queue:
+ cid = queue.popleft() # O(1) operation
+
+ relatives_set = _relatives(cid)
+ new_relatives = relatives_set - seen
+
+ if new_relatives:
+ # Add new relatives to queue and result if they pass predicate
+ for relative in new_relatives:
+ if predicate(graph.cells[relative]):
+ result.add(relative)
+ seen.add(relative)
+ queue.append(relative)
+
+ return result
+
+
+def induced_subgraph(
+ graph: GraphTopology, cell_ids: Collection[CellId_t]
+) -> tuple[dict[CellId_t, set[CellId_t]], dict[CellId_t, set[CellId_t]]]:
+ """Return parents and children for each node in `cell_ids`
+
+ Represents the subgraph induced by `cell_ids`.
+ """
+ parents: dict[CellId_t, set[CellId_t]] = {}
+ children: dict[CellId_t, set[CellId_t]] = {}
+ for cid in cell_ids:
+ parents[cid] = set(p for p in graph.parents[cid] if p in cell_ids)
+ children[cid] = set(c for c in graph.children[cid] if c in cell_ids)
+ return parents, children
+
+
+def get_cycles(
+ graph: DirectedGraph, cell_ids: Collection[CellId_t]
+) -> list[tuple[Edge, ...]]:
+ """Get all cycles among `cell_ids`."""
+ _, induced_children = induced_subgraph(graph, cell_ids)
+ induced_edges = set(
+ [(u, v) for u in induced_children for v in induced_children[u]]
+ )
+ return [c for c in graph.cycles if all(e in induced_edges for e in c)]
+
+
+def topological_sort(
+ graph: GraphTopology, cell_ids: Collection[CellId_t]
+) -> list[CellId_t]:
+ """Sort `cell_ids` in a topological order using a heap queue.
+
+ When multiple cells have the same parents (including no parents), the tie is broken by
+ registration order - cells registered earlier are processed first.
+ """
+ from heapq import heapify, heappop, heappush
+
+ # Use a list for O(1) lookup of registration order
+ registration_order = list(graph.cells.keys())
+ top_down_keys = {key: idx for idx, key in enumerate(registration_order)}
+
+ # Build adjacency lists and in-degree counts
+ parents, children = induced_subgraph(graph, cell_ids)
+ in_degree = {cid: len(parents[cid]) for cid in cell_ids}
+
+ # Initialize heap with roots
+ heap = [
+ (top_down_keys[cid], cid) for cid in cell_ids if in_degree[cid] == 0
+ ]
+ heapify(heap)
+
+ sorted_cell_ids: list[CellId_t] = []
+ while heap:
+ _, cid = heappop(heap)
+ sorted_cell_ids.append(cid)
+
+ # Process children
+ for child in children[cid]:
+ in_degree[child] -= 1
+ if in_degree[child] == 0:
+ heappush(heap, (top_down_keys[child], child))
+
+ return sorted_cell_ids
+
+
+def prune_cells_for_overrides(
+ graph: DirectedGraph,
+ execution_order: Collection[CellId_t],
+ overrides: dict[str, Any],
+) -> list[CellId_t]:
+ """Prune cells from execution when their definitions are overridden.
+
+ When variable definitions are provided externally (overrides), this function
+ identifies cells that would normally define those variables and removes them
+ from the execution order. It also validates that all definitions from pruned
+ cells are provided in the overrides.
+
+ Args:
+ graph: The dataflow graph containing cell dependencies
+ execution_order: Ordered collection of cells to execute
+ overrides: Dictionary mapping variable names to their override values
+
+ Returns:
+ Filtered execution order excluding cells whose definitions are overridden
+
+ Raises:
+ IncompleteRefsError: If overrides don't provide all definitions from
+ pruned cells (e.g., a cell defines both x and y, but only x is
+ provided in overrides)
+
+ Example:
+ If cell A defines variables x and y, and overrides = {"x": 1, "y": 2},
+ then cell A will be pruned from execution_order. However, if overrides
+ only contains {"x": 1}, an IncompleteRefsError is raised because y is
+ missing.
+ """
+ if not overrides:
+ return list(execution_order)
+
+ cells_to_prune: set[CellId_t] = set()
+ refs = set(overrides.keys())
+
+ # Find cells that define the overridden variables
+ for ref_name in refs:
+ if ref_name in graph.definitions:
+ defining_cells = graph.get_defining_cells(ref_name)
+ cells_to_prune.update(defining_cells)
+
+ # Validate that all definitions from pruned cells are provided
+ missing_defs: set[str] = set()
+ for cell_id in cells_to_prune:
+ cell = graph.cells[cell_id]
+ # Check all definitions this cell would have provided
+ for missing in cell.defs - refs:
+ missing_defs.add(missing)
+
+ if missing_defs:
+ from marimo._ast.errors import IncompleteRefsError
+
+ raise IncompleteRefsError(
+ f"When providing refs that override cell definitions, you must "
+ f"provide all definitions from those cells. Missing: {sorted(missing_defs)}. "
+ f"Provided refs: {sorted(refs)}."
+ )
+
+ # Return filtered execution order
+ return [cid for cid in execution_order if cid not in cells_to_prune]
+
+
+def get_import_block_relatives(
+ graph: DirectedGraph,
+) -> Callable[[CellId_t, bool], set[CellId_t]]:
+ def import_block_relatives(cid: CellId_t, children: bool) -> set[CellId_t]:
+ if not children:
+ return graph.parents[cid]
+
+ cell = graph.cells[cid]
+ if not cell.import_workspace.is_import_block:
+ return graph.children[cid]
+
+ # This cell is an import block, which should be special cased:
+ #
+ # We prune definitions that have already been imported from the set of
+ # definitions used to find the descendants of this cell.
+ unimported_defs = cell.defs - cell.import_workspace.imported_defs
+
+ children_ids = {
+ child_id
+ for name in unimported_defs
+ for child_id in graph.get_referring_cells(name, language="python")
+ }
+
+ # If children haven't been executed, then still use imported defs;
+ # handle an edge case when an import cell is interrupted by an
+ # exception or user interrupt, so that a module is imported but the
+ # cell's children haven't run.
+ if cell.import_workspace.imported_defs:
+ interrupted_states = {
+ "interrupted",
+ "cancelled",
+ "marimo-error",
+ None,
+ }
+ children_ids.update(
+ child_id
+ for name in cell.import_workspace.imported_defs
+ for child_id in graph.get_referring_cells(
+ name, language="python"
+ )
+ if graph.cells[child_id].run_result_status
+ in interrupted_states
+ )
+
+ return children_ids
+
+ return import_block_relatives
+
+
+__all__ = [
+ "Edge",
+ "EdgeWithVar",
+ "DirectedGraph",
+ "Runner",
+ "get_cycles",
+ "get_import_block_relatives",
+ "induced_subgraph",
+ "prune_cells_for_overrides",
+ "topological_sort",
+ "transitive_closure",
+]
diff --git a/marimo/_runtime/dataflow/cycles.py b/marimo/_runtime/dataflow/cycles.py
new file mode 100644
index 0000000000..41ee9f2358
--- /dev/null
+++ b/marimo/_runtime/dataflow/cycles.py
@@ -0,0 +1,91 @@
+# Copyright 2024 Marimo. All rights reserved.
+"""Cycle detection and tracking for cell dependencies."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from marimo._runtime.dataflow.types import Edge
+
+if TYPE_CHECKING:
+ from collections.abc import Collection
+
+ from marimo._runtime.dataflow.topology import GraphTopology
+ from marimo._types.ids import CellId_t
+
+
+@dataclass
+class CycleTracker:
+ """Detects and tracks cycles in the graph.
+
+ Responsibilities:
+ - Detect cycles when edges are added
+ - Maintain cycle set
+ - Remove broken cycles
+ """
+
+ # The set of cycles in the graph
+ # Each cycle is represented as a tuple of edges
+ cycles: set[tuple[Edge, ...]] = field(default_factory=set)
+
+ def detect_cycle_for_edge(
+ self,
+ edge: Edge,
+ topology: GraphTopology,
+ ) -> tuple[Edge, ...] | None:
+ """Detect if adding an edge creates a cycle.
+
+ Args:
+ edge: The edge (parent, child) being added
+ topology: The graph topology to search for paths
+
+ Returns:
+ The cycle as a tuple of edges if one exists, None otherwise.
+ The cycle includes the new edge plus the path from child back to parent.
+ """
+ parent, child = edge
+ # Check if there's a path from child back to parent
+ # If so, adding this edge creates a cycle
+ path = topology.get_path(child, parent)
+ if path:
+ # The cycle is: edge + path
+ cycle = tuple([edge] + path)
+ self.cycles.add(cycle)
+ return cycle
+ return None
+
+ def remove_cycles_with_edge(self, edge: Edge) -> None:
+ """Remove all cycles that contain the given edge.
+
+ This should be called when an edge is removed from the graph.
+ """
+ broken_cycles = [c for c in self.cycles if edge in c]
+ for c in broken_cycles:
+ self.cycles.remove(c)
+
+ def get_cycles(
+ self,
+ cell_ids: Collection[CellId_t],
+ topology: GraphTopology,
+ ) -> list[tuple[Edge, ...]]:
+ """Get all cycles among the given cell_ids.
+
+ Args:
+ cell_ids: The cells to consider
+ topology: The graph topology
+
+ Returns:
+ List of cycles, where each cycle is a tuple of edges.
+ Only returns cycles where all edges are between cells in cell_ids.
+ """
+ # Build induced subgraph edges
+ induced_edges = set()
+ for u in cell_ids:
+ if u in topology.children:
+ for v in topology.children[u]:
+ if v in cell_ids:
+ induced_edges.add((u, v))
+
+ # Filter cycles to those in the induced subgraph
+ return [c for c in self.cycles if all(e in induced_edges for e in c)]
diff --git a/marimo/_runtime/dataflow/definitions.py b/marimo/_runtime/dataflow/definitions.py
new file mode 100644
index 0000000000..8f1e71576c
--- /dev/null
+++ b/marimo/_runtime/dataflow/definitions.py
@@ -0,0 +1,153 @@
+# Copyright 2024 Marimo. All rights reserved.
+"""Variable definition tracking for cells."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from marimo._ast.sql_visitor import SQLRef
+ from marimo._ast.visitor import Name, VariableData
+ from marimo._types.ids import CellId_t
+
+
+@dataclass
+class DefinitionRegistry:
+ """Tracks variable definitions across cells.
+
+ Responsibilities:
+ - Track which cells define which variables
+ - Handle typed definitions (SQL vs Python)
+ - Detect multiply-defined names
+ - SQL hierarchical reference matching
+ """
+
+ # A mapping from defs to the cells that define them
+ definitions: dict[Name, set[CellId_t]] = field(default_factory=dict)
+
+ # Typed definitions for SQL support: (name, type) -> cell_ids
+ # e.g. ("my_table", "table") -> {cell_id_1}
+ typed_definitions: dict[tuple[Name, str], set[CellId_t]] = field(
+ default_factory=dict
+ )
+
+ # Track all types for a given definition name
+ # e.g. "my_table" -> {"table", "view"}
+ definition_types: dict[Name, set[str]] = field(default_factory=dict)
+
+ def register_definition(
+ self,
+ cell_id: CellId_t,
+ name: Name,
+ variable_data: list[VariableData],
+ ) -> set[CellId_t]:
+ """Register a definition for a cell.
+
+ Args:
+ cell_id: The cell that defines the variable
+ name: The variable name
+ variable_data: List of metadata about the variable (from cell.variable_data[name])
+
+ Returns:
+ Set of sibling cell IDs (cells that also define this variable)
+ """
+ variable = variable_data[-1] # Only the last definition matters
+ typed_def = (name, variable.kind)
+
+ # Check if this is a duplicate definition
+ if (
+ name in self.definitions
+ and typed_def not in self.typed_definitions
+ ):
+ # Duplicate if the qualified name is no different
+ if variable.qualified_name == name or variable.language != "sql":
+ self.definitions[name].add(cell_id)
+ else:
+ self.definitions.setdefault(name, set()).add(cell_id)
+
+ self.typed_definitions.setdefault(typed_def, set()).add(cell_id)
+ self.definition_types.setdefault(name, set()).add(variable.kind)
+
+ # Return siblings (other cells that define this name)
+ siblings = self.definitions[name] - {cell_id}
+ return siblings
+
+ def unregister_definitions(
+ self,
+ cell_id: CellId_t,
+ defs: set[Name],
+ ) -> None:
+ """Unregister all definitions for a cell.
+
+ Args:
+ cell_id: The cell being removed
+ defs: The set of variable names defined by the cell
+ """
+ for name in defs:
+ if name not in self.definitions:
+ continue
+
+ name_defs = self.definitions[name]
+ name_defs.discard(cell_id)
+
+ if not name_defs:
+ # No more cells define this name, so we remove it
+ del self.definitions[name]
+ # Clean up all typed definitions
+ for typed_def in self.definition_types.get(name, set()):
+ self.typed_definitions.pop((name, typed_def), None)
+ self.definition_types.pop(name, None)
+
+ def get_defining_cells(self, name: Name) -> set[CellId_t]:
+ """Get all cells that define a variable name.
+
+ This is a singleton for well-formed graphs (no multiply-defined names).
+
+ Args:
+ name: The variable name
+
+ Returns:
+ Set of cell IDs that define this name
+ """
+ return self.definitions.get(name, set())
+
+ def find_sql_hierarchical_matches(
+ self, sql_ref: SQLRef
+ ) -> list[tuple[set[CellId_t], Name]]:
+ """Find cells that define components of a hierarchical SQL reference.
+
+ This method searches through all definitions in the graph to find cells
+ that define the individual components (table, schema, or catalog) of the
+ hierarchical reference.
+
+ For example, given a reference "my_schema.my_table", this method will:
+ - Look for cells that define a table/view named "my_table"
+ - Look for cells that define a catalog named "my_schema"
+ (when the reference has at least 2 parts)
+
+ Args:
+ sql_ref: A hierarchical SQL reference (e.g., "schema.table",
+ "catalog.schema.table") to find matching definitions for.
+
+ Returns:
+ A list of tuples containing:
+ - A set of cell IDs that define components of the hierarchical reference
+ - The definition of the name that was found (e.g., "schema.table" -> "table")
+ """
+ matching_cell_ids_list = []
+
+ for (def_name, kind), cell_ids in self.typed_definitions.items():
+ # Match table/view definitions
+ if sql_ref.contains_hierarchical_ref(def_name, kind):
+ matching_cell_ids_list.append((cell_ids, def_name))
+
+ return matching_cell_ids_list
+
+ def get_multiply_defined(self) -> list[Name]:
+ """Return a list of names that are defined in multiple cells."""
+ names: list[Name] = []
+ for name, definers in self.definitions.items():
+ if len(definers) > 1:
+ names.append(name)
+ return names
diff --git a/marimo/_runtime/dataflow/edges.py b/marimo/_runtime/dataflow/edges.py
new file mode 100644
index 0000000000..34f0d6308b
--- /dev/null
+++ b/marimo/_runtime/dataflow/edges.py
@@ -0,0 +1,269 @@
+# Copyright 2024 Marimo. All rights reserved.
+"""Edge computation logic for cell dependencies."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Literal, Optional, cast
+
+from marimo import _loggers
+
+if TYPE_CHECKING:
+ from marimo._ast.cell import CellImpl
+ from marimo._ast.sql_visitor import SQLRef
+ from marimo._ast.visitor import Name
+ from marimo._runtime.dataflow.definitions import DefinitionRegistry
+ from marimo._runtime.dataflow.topology import GraphTopology
+ from marimo._types.ids import CellId_t
+
+from marimo._ast.sql_visitor import SQLTypes
+
+LOGGER = _loggers.marimo_logger()
+
+
+class EdgeComputer:
+ """Computes graph edges based on cell definitions and references.
+
+ Responsibilities:
+ - Compute parent/child relationships from defs/refs
+ - Handle SQL reference resolution
+ - Handle deleted reference semantics
+ - Apply language-specific edge rules (Python/SQL)
+ """
+
+ def __init__(
+ self,
+ topology: GraphTopology,
+ definitions: DefinitionRegistry,
+ ):
+ self.topology = topology
+ self.definitions = definitions
+
+ def get_referring_cells(
+ self, name: Name, language: Literal["python", "sql"]
+ ) -> set[CellId_t]:
+ """Get all cells that have a ref to `name`.
+
+ The variable can be either a Python variable or a SQL variable (table).
+ SQL variables don't leak to Python cells, but Python variables do leak
+ to SQL.
+
+ Only does a local analysis of refs, without taking into consideration
+ whether refs are defined by other cells.
+ """
+ if language == "sql":
+ # For SQL, only return SQL cells that reference the name
+ cells = set()
+ for cid, cell in self.topology.cells.items():
+ if cell.language != "sql":
+ continue
+
+ for ref in cell.refs:
+ # Direct reference match
+ if ref == name:
+ cells.add(cid)
+ break
+
+ sql_ref = cell.sql_refs.get(ref)
+
+ kind: SQLTypes = "any"
+ if name in cell.variable_data:
+ variable_data = cell.variable_data[name][-1]
+ kind = cast(SQLTypes, variable_data.kind)
+
+ # Hierarchical reference match
+ if sql_ref and sql_ref.matches_hierarchical_ref(
+ name, ref, kind
+ ):
+ cells.add(cid)
+ break
+
+ return cells
+ else:
+ # For Python, return all cells that reference the name
+ return {
+ cid
+ for cid, cell in self.topology.cells.items()
+ if name in cell.refs
+ }
+
+ def _is_valid_cell_reference(
+ self, cell_id: CellId_t, variable_name: Name
+ ) -> bool:
+ """Check if a cell reference is valid and log errors if not."""
+ if cell_id not in self.topology.cells:
+ LOGGER.error(
+ "Variable %s is defined in cell %s, but is not in the graph",
+ variable_name,
+ cell_id,
+ )
+ return False
+ return True
+
+ def _resolve_variable_name(
+ self,
+ name: Name,
+ other_cell: CellImpl,
+ sql_ref: Optional[SQLRef],
+ sql_matches: list[tuple[set[CellId_t], Name]],
+ ) -> Name:
+ """Resolve the variable name to use when checking if it exists in another cell.
+
+ For regular (non-SQL) references, returns the original name unchanged.
+ For SQL hierarchical references, finds the variable name from sql_matches that
+ is actually defined in the other_cell.
+
+ Example:
+ cell_1: CREATE SCHEMA schema_name
+ cell_2: CREATE TABLE schema_name.table_name
+ cell_3: FROM schema_name.table_name SELECT *
+
+ When cell_3 references "schema_name.table_name":
+ - For cell_1: returns "schema_name" (what cell_1 actually defines)
+ - For cell_2: returns "table_name" (what cell_2 actually defines)
+ """
+ if not sql_ref or name in other_cell.variable_data:
+ return name
+
+ # For SQL hierarchical references, find the matching variable name
+ for _, matching_variable_name in sql_matches:
+ if matching_variable_name in other_cell.variable_data:
+ return matching_variable_name
+
+ return name
+
+ def compute_edges_for_cell(
+ self, cell_id: CellId_t, cell: CellImpl
+ ) -> tuple[set[CellId_t], set[CellId_t]]:
+ """Compute parent and child edges for a cell being registered.
+
+ Args:
+ cell_id: The ID this cell is being registered with
+ cell: The cell to compute edges for
+
+ Returns:
+ Tuple of (parents, children) where:
+ - parents: cells that this cell depends on
+ - children: cells that depend on this cell
+ """
+ parents: set[CellId_t] = set()
+ children: set[CellId_t] = set()
+
+ # First, process the variables that this cell defines. Any cell
+ # that refers to a defined variable becomes a child of this cell.
+ for name, variable_data in cell.variable_data.items():
+ # NB. Only the last definition matters.
+ # Technically more nuanced with branching statements, but this is
+ # the best we can do with static analysis.
+ variable = variable_data[-1]
+
+ # a cell can refer to its own defs, but that doesn't add an
+ # edge to the dependency graph
+ referring_cells = self.get_referring_cells(
+ name,
+ language=variable.language,
+ ) - {cell_id}
+
+ children.update(referring_cells)
+
+ # Next, process the cells references. The cell becomes a child
+ # of cells that define its referenced variables. We also have
+ # special logic for handling references that are deleted by this cell,
+ # since cells that delete variables that were defined elsewhere
+ # are made children of cells that reference that variable.
+ for name in cell.refs:
+ # First, for each referenced variable, we add cells that define
+ # that variable as parents
+ other_ids_defining_name: set[CellId_t] = (
+ self.definitions.definitions.get(name, set()) - {cell_id}
+ )
+
+ variable_name: Name = name
+
+ # Handle SQL matching for hierarchical references
+ sql_matches: list[tuple[set[CellId_t], Name]] = []
+ sql_ref = cell.sql_refs.get(name)
+ if sql_ref:
+ sql_matches = self.definitions.find_sql_hierarchical_matches(
+ sql_ref
+ )
+ for matching_cell_ids, _ in sql_matches:
+ if cell_id in matching_cell_ids:
+ LOGGER.debug("Cell %s is referencing itself", cell_id)
+ continue
+ other_ids_defining_name.update(matching_cell_ids)
+
+ # If other_ids_defining_name is empty, the user will get a
+ # NameError at runtime (unless the symbol is a builtin).
+ for other_id in other_ids_defining_name:
+ if other_id == cell_id:
+ LOGGER.error("Cell %s is referencing itself", cell_id)
+ continue
+ if not self._is_valid_cell_reference(other_id, variable_name):
+ continue
+ other_cell = self.topology.cells[other_id]
+
+ variable_name = self._resolve_variable_name(
+ variable_name, other_cell, sql_ref, sql_matches
+ )
+
+ # If we don't have a matching variable name, skip
+ if variable_name not in other_cell.variable_data:
+ LOGGER.error(
+ "Variable %s is not defined in cell %s",
+ variable_name,
+ other_id,
+ )
+ continue
+
+ other_variable_data = other_cell.variable_data[variable_name][
+ -1
+ ]
+ language = other_variable_data.language
+ if language == "sql" and cell.language == "python":
+ # SQL table/db def -> Python ref is not an edge
+ continue
+ if language == "sql" and cell.language == "sql":
+ # Edges between SQL cells need to respect hierarchy.
+ if sql_ref and not sql_ref.matches_hierarchical_ref(
+ variable_name,
+ other_variable_data.qualified_name or name,
+ kind=cast(SQLTypes, other_variable_data.kind),
+ ):
+ continue
+ parents.add(other_id)
+
+ # Next, any cell that deletes this referenced variable is made
+ # a child of this cell. In particular, if a cell deletes a
+ # variable, it becomes a child of all other cells that
+ # reference that variable. This means that if two cells delete
+ # the same variable, they form a cycle.
+ #
+ # For example, two cells
+ #
+ # cell u: x
+ # cell v: del x
+ #
+ # v becomes a child of u.
+ #
+ # Another example:
+ #
+ # cell u: del x
+ # cell v: del x
+ #
+ # u and v form a cycle.
+ other_ids_deleting_name: set[CellId_t] = {
+ cid
+ for cid in self.get_referring_cells(name, language="python")
+ if name in self.topology.cells[cid].deleted_refs
+ } - {cell_id}
+ children.update(other_ids_deleting_name)
+
+ # Finally, if this cell deletes a variable, we make it a child of
+ # all other cells that reference this variable.
+ for name in cell.deleted_refs:
+ referring_cells = self.get_referring_cells(
+ name, language="python"
+ ) - {cell_id}
+ parents.update(referring_cells)
+
+ return parents, children
diff --git a/marimo/_runtime/dataflow/graph.py b/marimo/_runtime/dataflow/graph.py
new file mode 100644
index 0000000000..09aa71f5f2
--- /dev/null
+++ b/marimo/_runtime/dataflow/graph.py
@@ -0,0 +1,442 @@
+# Copyright 2024 Marimo. All rights reserved.
+"""Graph coordinator that orchestrates all dataflow components."""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Callable, Literal, Optional
+
+from marimo import _loggers
+from marimo._ast.compiler import code_key
+from marimo._ast.variables import is_mangled_local
+from marimo._runtime.dataflow.cycles import CycleTracker
+from marimo._runtime.dataflow.definitions import DefinitionRegistry
+from marimo._runtime.dataflow.edges import EdgeComputer
+from marimo._runtime.dataflow.topology import (
+ GraphTopology,
+ MutableGraphTopology,
+)
+
+if TYPE_CHECKING:
+ from collections.abc import Mapping
+
+ from marimo._ast.cell import CellImpl
+ from marimo._ast.visitor import ImportData, Name, VariableData
+ from marimo._types.ids import CellId_t
+
+LOGGER = _loggers.marimo_logger()
+
+
+@dataclass(frozen=True)
+class DirectedGraph(GraphTopology):
+ """Main entry point that coordinates all graph operations.
+
+ Responsibilities:
+ - Coordinate topology, definitions, cycles, siblings
+ - Execute register_cell/delete_cell operations
+ - Maintain thread safety
+ - Delegate to specialists
+ """
+
+ topology: MutableGraphTopology = field(
+ default_factory=MutableGraphTopology
+ )
+ definition_registry: DefinitionRegistry = field(
+ default_factory=DefinitionRegistry
+ )
+ cycle_tracker: CycleTracker = field(default_factory=CycleTracker)
+
+ # Cells that define the same name
+ #
+ # siblings[cell_id] is a set of cell ids, one for each cell that shares a
+ # definition with cell_id.
+ #
+ # If this dict is non-empty, then the marimo program contains multiply
+ # defined names (and is therefore in an error state)
+ # TODO: remove this field (might be unused)
+ siblings: dict[CellId_t, set[CellId_t]] = field(default_factory=dict)
+
+ # This lock must be acquired during methods that mutate the graph; it's
+ # only needed because a graph is shared between the kernel and the code
+ # completion service. It should almost always be uncontended.
+ lock: threading.Lock = field(default_factory=threading.Lock)
+
+ def is_cell_cached(self, cell_id: CellId_t, code: str) -> bool:
+ """Whether a cell with id `cell_id` and code `code` is in the graph."""
+ return (
+ cell_id in self.topology.cells
+ and code_key(code) == self.topology.cells[cell_id].key
+ )
+
+ def get_defining_cells(self, name: Name) -> set[CellId_t]:
+ """Get all cells that define name.
+
+ This is a singleton for well-formed graphs.
+ """
+ return self.definition_registry.get_defining_cells(name)
+
+ def get_referring_cells(
+ self, name: Name, language: Literal["python", "sql"]
+ ) -> set[CellId_t]:
+ """Get all cells that have a ref to `name`.
+
+ The variable can be either a Python variable or a SQL variable (table).
+ SQL variables don't leak to Python cells, but Python variables do leak
+ to SQL.
+
+ Only does a local analysis of refs, without taking into consideration
+ whether refs are defined by other cells.
+ """
+ edge_computer = EdgeComputer(self.topology, self.definition_registry)
+ return edge_computer.get_referring_cells(name, language)
+
+ def register_cell(self, cell_id: CellId_t, cell: CellImpl) -> None:
+ """Add a cell to the graph.
+
+ Mutates the graph, acquiring `self.lock`.
+
+ Requires that `cell_id` is not already in the graph.
+ """
+ LOGGER.debug("Acquiring graph lock to register cell %s", cell_id)
+ with self.lock:
+ LOGGER.debug("Acquired graph lock.")
+ assert cell_id not in self.topology.cells
+
+ # Add the cell to topology
+ self.topology.add_node(cell_id, cell)
+
+ # Initialize siblings for this cell
+ self.siblings[cell_id] = set()
+
+ # Process definitions and build sibling relationships FIRST
+ # This must happen before computing edges because edge computation
+ # needs to look up definitions
+ for name, variable_data in cell.variable_data.items():
+ siblings = self.definition_registry.register_definition(
+ cell_id, name, variable_data
+ )
+ # Update siblings bidirectionally
+ for sibling in siblings:
+ self.siblings[cell_id].add(sibling)
+ self.siblings[sibling].add(cell_id)
+
+ # Now compute edges using EdgeComputer (which can now find the definitions)
+ edge_computer = EdgeComputer(
+ self.topology, self.definition_registry
+ )
+ parents, children = edge_computer.compute_edges_for_cell(
+ cell_id, cell
+ )
+
+ # Add edges to topology
+ for parent_id in parents:
+ self.topology.add_edge(parent_id, cell_id)
+ # Detect cycle for this edge
+ self.cycle_tracker.detect_cycle_for_edge(
+ (parent_id, cell_id), self.topology
+ )
+
+ for child_id in children:
+ self.topology.add_edge(cell_id, child_id)
+ # Detect cycle for this edge
+ self.cycle_tracker.detect_cycle_for_edge(
+ (cell_id, child_id), self.topology
+ )
+
+ LOGGER.debug("Registered cell %s and released graph lock", cell_id)
+ if self.is_any_ancestor_stale(cell_id):
+ self.set_stale({cell_id})
+
+ if self.is_any_ancestor_disabled(cell_id):
+ cell.set_runtime_state(status="disabled-transitively")
+
+ def is_any_ancestor_stale(self, cell_id: CellId_t) -> bool:
+ """Check if any ancestor of a cell is stale."""
+ return any(
+ self.topology.cells[cid].stale for cid in self.ancestors(cell_id)
+ )
+
+ def is_any_ancestor_disabled(self, cell_id: CellId_t) -> bool:
+ """Check if any ancestor of a cell is disabled."""
+ return any(
+ self.topology.cells[cid].config.disabled
+ for cid in self.ancestors(cell_id)
+ )
+
+ def disable_cell(self, cell_id: CellId_t) -> None:
+ """Disables a cell in the graph.
+
+ Does not mutate the graph (but does mutate cell statuses).
+
+ Returns the ids of descendants that are disabled transitively.
+ """
+ if cell_id not in self.topology.cells:
+ raise ValueError(f"Cell {cell_id} not found")
+
+ from marimo._runtime.dataflow import transitive_closure
+
+ for cid in transitive_closure(self, {cell_id}) - {cell_id}:
+ cell = self.topology.cells[cid]
+ cell.set_runtime_state(status="disabled-transitively")
+
+ def enable_cell(self, cell_id: CellId_t) -> set[CellId_t]:
+ """Enables a cell in the graph.
+
+ Does not mutate the graph (but does mutate cell statuses).
+
+ Returns:
+ - set of cells that were stale and should be re-run
+ """
+ if cell_id not in self.topology.cells:
+ raise ValueError(f"Cell {cell_id} not found")
+
+ from marimo._runtime.dataflow import transitive_closure
+
+ cells_to_run: set[CellId_t] = set()
+ for cid in transitive_closure(self, {cell_id}):
+ if not self.is_disabled(cid):
+ child = self.topology.cells[cid]
+ if child.stale:
+ # cell was previously disabled, is no longer
+ # disabled, and is stale: needs to run.
+ cells_to_run.add(cid)
+ if child.disabled_transitively:
+ # cell is no longer disabled: status -> idle
+ child.set_runtime_state("idle")
+ return cells_to_run
+
+ def delete_cell(self, cell_id: CellId_t) -> set[CellId_t]:
+ """Removes a cell from the graph.
+
+ Mutates the graph, acquiring `self.lock`.
+
+ Returns the ids of the children of the removed cell.
+ """
+ LOGGER.debug("Acquiring graph lock to delete cell %s", cell_id)
+ with self.lock:
+ LOGGER.debug("Acquired graph lock to delete cell %s", cell_id)
+ if cell_id not in self.topology.cells:
+ raise ValueError(f"Cell {cell_id} not found")
+
+ # Grab a reference to children before we remove it
+ children = self.topology.children[cell_id].copy()
+
+ # Removing this cell from its defs' definer sets
+ cell = self.topology.cells[cell_id]
+ self.definition_registry.unregister_definitions(cell_id, cell.defs)
+
+ # Remove cycles that are broken from removing this cell
+ edges = [
+ (cell_id, child) for child in self.topology.children[cell_id]
+ ] + [
+ (parent, cell_id) for parent in self.topology.parents[cell_id]
+ ]
+ for e in edges:
+ self.cycle_tracker.remove_cycles_with_edge(e)
+
+ # Remove from siblings
+ for sibling_id in self.siblings.get(cell_id, set()):
+ self.siblings[sibling_id].discard(cell_id)
+ self.siblings.pop(cell_id, None)
+
+ # Purge this cell from the graph topology
+ self.topology.remove_node(cell_id)
+
+ LOGGER.debug("Deleted cell %s and Released graph lock.", cell_id)
+ return children
+
+ def is_disabled(self, cell_id: CellId_t) -> bool:
+ """Check if a cell is disabled (directly or transitively)."""
+ if cell_id not in self.topology.cells:
+ raise ValueError(f"Cell {cell_id} not in graph.")
+ cell = self.topology.cells[cell_id]
+ if cell.config.disabled:
+ return True
+ seen: set[CellId_t] = set()
+ queue = [cell_id]
+ while queue:
+ cid = queue.pop()
+ seen.add(cid)
+ for parent_id in self.topology.parents[cid]:
+ if parent_id in seen:
+ continue
+ elif self.topology.cells[parent_id].config.disabled:
+ return True
+ else:
+ queue.append(parent_id)
+ return False
+
+ def get_imports(
+ self, cell_id: Optional[CellId_t] = None
+ ) -> dict[Name, ImportData]:
+ """Get imports from cell(s)."""
+ imports = {}
+ cells = (
+ self.topology.cells.values()
+ if cell_id is None
+ else [self.topology.cells[cell_id]]
+ )
+ for cell in cells:
+ for imported in cell.imports:
+ imports[imported.definition] = imported
+ return imports
+
+ def get_multiply_defined(self) -> list[Name]:
+ """Return a list of names that are defined in multiple cells."""
+ return self.definition_registry.get_multiply_defined()
+
+ def get_deleted_nonlocal_ref(self) -> list[Name]:
+ """Get names that are deleted but defined elsewhere."""
+ names: list[Name] = []
+ for cell in self.topology.cells.values():
+ for ref in cell.deleted_refs:
+ if ref in self.definition_registry.definitions:
+ names.append(ref)
+ return names
+
+ def set_stale(
+ self, cell_ids: set[CellId_t], prune_imports: bool = False
+ ) -> None:
+ """Mark cells as stale (need re-execution)."""
+ from marimo._runtime.dataflow import (
+ get_import_block_relatives,
+ transitive_closure,
+ )
+
+ relatives = (
+ None if not prune_imports else get_import_block_relatives(self)
+ )
+
+ for cid in transitive_closure(self, cell_ids, relatives=relatives):
+ self.topology.cells[cid].set_stale(stale=True)
+
+ def get_stale(self) -> set[CellId_t]:
+ """Get all stale cells."""
+ return {cid for cid, cell in self.topology.cells.items() if cell.stale}
+
+ def get_transitive_references(
+ self,
+ refs: set[Name],
+ inclusive: bool = True,
+ predicate: Callable[[Name, VariableData], bool] | None = None,
+ ) -> set[Name]:
+ """Return a set of the passed-in cells' references and their
+ references on the block (function / class) level.
+
+ If inclusive, includes the references of the passed-in cells in the
+ set.
+
+ If predicate, only references satisfying predicate(ref) are included
+ """
+ # TODO: Consider caching on the graph level and updating on register /
+ # delete
+ processed: set[Name] = set()
+ queue: set[Name] = refs & self.definition_registry.definitions.keys()
+ predicate = predicate or (lambda *_: True)
+
+ while queue:
+ # Should ideally be one cell per ref, but for completion, stay
+ # agnostic to potenital cycles.
+ cells = {
+ cell_id
+ for ref in queue
+ for cell_id in self.definition_registry.definitions.get(
+ ref, set()
+ )
+ }
+
+ for cell_id in cells:
+ data = self.topology.cells[cell_id].variable_data
+ newly_processed = set(data.keys()) & queue
+ processed.update(newly_processed)
+ queue.difference_update(newly_processed)
+ for variable in newly_processed:
+ # variables can be defined multiple times in a single
+ # cell ...
+ for datum in data[variable]:
+ if predicate(variable, datum):
+ to_process = datum.required_refs - processed
+ queue.update(
+ to_process
+ & self.definition_registry.definitions.keys()
+ )
+ # Private variables referenced by public functions
+ # have to be included.
+ processed.update(
+ maybe_private
+ for maybe_private in (
+ to_process
+ - self.definition_registry.definitions.keys()
+ )
+ if is_mangled_local(maybe_private, cell_id)
+ )
+
+ if inclusive:
+ return processed | refs
+ return processed - refs
+
+ def copy(self, filename: None | str = None) -> DirectedGraph:
+ """Return a deep copy of the graph by recompiling all cells.
+
+ This is mainly useful in the case where recompilation must be done
+ due to a dynamically changing notebook, where the line cache must be
+ consistent with the cell code, e.g. for debugging.
+ """
+ from marimo._ast.compiler import compile_cell
+
+ graph = DirectedGraph()
+ with self.lock:
+ for cid, old_cell in self.topology.cells.items():
+ cell = compile_cell(
+ old_cell.code,
+ cell_id=cid,
+ filename=filename,
+ )
+ # Carry over import data manually
+ imported_defs = old_cell.import_workspace.imported_defs
+ is_import_block = old_cell.import_workspace.is_import_block
+ cell.import_workspace.imported_defs = imported_defs
+ cell.import_workspace.is_import_block = is_import_block
+ # Reregister
+ graph.register_cell(cid, cell)
+ return graph
+
+ @property
+ def cells(self) -> Mapping[CellId_t, CellImpl]:
+ """Get the cells dictionary."""
+ return self.topology.cells
+
+ @property
+ def parents(self) -> Mapping[CellId_t, set[CellId_t]]:
+ """Get the parents dictionary."""
+ return self.topology.parents
+
+ @property
+ def children(self) -> Mapping[CellId_t, set[CellId_t]]:
+ """Get the children dictionary."""
+ return self.topology.children
+
+ def get_path(
+ self, source: CellId_t, dst: CellId_t
+ ) -> list[tuple[CellId_t, CellId_t]]:
+ """Get a path from `source` to `dst`, if any."""
+ return self.topology.get_path(source, dst)
+
+ def descendants(self, cell_id: CellId_t) -> set[CellId_t]:
+ """Get all descendants of a cell."""
+ return self.topology.descendants(cell_id)
+
+ def ancestors(self, cell_id: CellId_t) -> set[CellId_t]:
+ """Get all ancestors of a cell."""
+ return self.topology.ancestors(cell_id)
+
+ @property
+ def definitions(self) -> Mapping[Name, set[CellId_t]]:
+ """Get the definitions dictionary."""
+ return self.definition_registry.definitions
+
+ @property
+ def cycles(self) -> set[tuple[tuple[CellId_t, CellId_t], ...]]:
+ """Get the cycles set."""
+ return self.cycle_tracker.cycles
diff --git a/marimo/_runtime/dataflow/runner.py b/marimo/_runtime/dataflow/runner.py
new file mode 100644
index 0000000000..804c86deae
--- /dev/null
+++ b/marimo/_runtime/dataflow/runner.py
@@ -0,0 +1,154 @@
+# Copyright 2024 Marimo. All rights reserved.
+"""Runner utility for executing individual cells in a graph."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from marimo._runtime.executor import ExecutionConfig, get_executor
+
+if TYPE_CHECKING:
+ from marimo._ast.cell import CellImpl
+ from marimo._runtime.dataflow.graph import DirectedGraph
+ from marimo._types.ids import CellId_t
+
+
+class Runner:
+ """Utility for running individual cells in a graph
+
+ This class provides methods to a run a cell in the graph and obtain its
+ output (last expression) and the values of its defs.
+
+ If needed, the runner will recursively compute the values of the cell's
+ refs by executing its ancestors. Refs can also be substituted by the
+ caller.
+
+ TODO(akshayka): Add an API for caching defs across cell runs.
+ """
+
+ def __init__(self, graph: DirectedGraph) -> None:
+ self._graph = graph
+ self._executor = get_executor(ExecutionConfig())
+
+ @staticmethod
+ def _returns(cell_impl: CellImpl, glbls: dict[str, Any]) -> dict[str, Any]:
+ return {name: glbls[name] for name in cell_impl.defs if name in glbls}
+
+ @staticmethod
+ def _substitute_refs(
+ cell_impl: CellImpl,
+ glbls: dict[str, Any],
+ kwargs: dict[str, Any],
+ ) -> None:
+ for argname, argvalue in kwargs.items():
+ if argname in cell_impl.refs:
+ glbls[argname] = argvalue
+ else:
+ raise ValueError(
+ f"Cell got unexpected argument {argname}"
+ f"The allowed arguments are {cell_impl.refs}."
+ )
+
+ def _get_ancestors(
+ self, cell_impl: CellImpl, kwargs: dict[str, Any]
+ ) -> set[CellId_t]:
+ from marimo._runtime.dataflow import transitive_closure
+
+ # Get the transitive closure of parents defining unsubstituted refs
+ graph = self._graph
+ substitutions = set(kwargs.keys())
+ unsubstituted_refs = cell_impl.refs - substitutions
+ parent_ids = set(
+ [
+ parent_id
+ for parent_id in graph.parents[cell_impl.cell_id]
+ if graph.cells[parent_id].defs.intersection(unsubstituted_refs)
+ ]
+ )
+ return transitive_closure(graph, parent_ids, children=False)
+
+ @staticmethod
+ def _validate_kwargs(cell_impl: CellImpl, kwargs: dict[str, Any]) -> None:
+ for argname in kwargs:
+ if argname not in cell_impl.refs:
+ raise ValueError(
+ f"Cell got unexpected argument {argname}; "
+ f"The allowed arguments are {cell_impl.refs}."
+ )
+
+ def is_coroutine(self, cell_id: CellId_t) -> bool:
+ return self._graph.cells[cell_id].is_coroutine() or any(
+ self._graph.cells[cid].is_coroutine()
+ for cid in self._get_ancestors(
+ self._graph.cells[cell_id], kwargs={}
+ )
+ )
+
+ async def run_cell_async(
+ self, cell_id: CellId_t, kwargs: dict[str, Any]
+ ) -> tuple[Any, dict[str, Any]]:
+ """Run a possibly async cell and its ancestors
+
+ Substitutes kwargs as refs for the cell, omitting ancestors that
+ whose refs are substituted.
+ """
+ from marimo._runtime.dataflow import topological_sort
+
+ graph = self._graph
+ cell_impl = graph.cells[cell_id]
+ Runner._validate_kwargs(cell_impl, kwargs)
+ ancestor_ids = self._get_ancestors(cell_impl, kwargs)
+
+ glbls: dict[str, Any] = {}
+ for cid in topological_sort(graph, ancestor_ids):
+ await self._executor.execute_cell_async(
+ graph.cells[cid], glbls, graph
+ )
+
+ Runner._substitute_refs(cell_impl, glbls, kwargs)
+ output = await self._executor.execute_cell_async(
+ graph.cells[cell_impl.cell_id], glbls, graph
+ )
+ defs = Runner._returns(cell_impl, glbls)
+ return output, defs
+
+ def run_cell_sync(
+ self, cell_id: CellId_t, kwargs: dict[str, Any]
+ ) -> tuple[Any, dict[str, Any]]:
+ """Run a synchronous cell and its ancestors
+
+ Substitutes kwargs as refs for the cell, omitting ancestors that
+ whose refs are substituted.
+
+ Raises a `RuntimeError` if the cell or any of its unsubstituted
+ ancestors are coroutine functions.
+ """
+ from marimo._runtime.dataflow import topological_sort
+
+ graph = self._graph
+ cell_impl = graph.cells[cell_id]
+ if cell_impl.is_coroutine():
+ raise RuntimeError(
+ "A coroutine function can't be run synchronously. "
+ "Use `run_async()` instead"
+ )
+
+ Runner._validate_kwargs(cell_impl, kwargs)
+ ancestor_ids = self._get_ancestors(cell_impl, kwargs)
+
+ if any(graph.cells[cid].is_coroutine() for cid in ancestor_ids):
+ raise RuntimeError(
+ "Cell has an ancestor that is a "
+ "coroutine (async) cell. Use `run_async()` instead"
+ )
+
+ glbls: dict[str, Any] = {}
+ for cid in topological_sort(graph, ancestor_ids):
+ self._executor.execute_cell(graph.cells[cid], glbls, graph)
+
+ self._substitute_refs(cell_impl, glbls, kwargs)
+ output = self._executor.execute_cell(
+ graph.cells[cell_impl.cell_id], glbls, graph
+ )
+ defs = Runner._returns(cell_impl, glbls)
+ return output, defs
diff --git a/marimo/_runtime/dataflow/topology.py b/marimo/_runtime/dataflow/topology.py
new file mode 100644
index 0000000000..ff91164142
--- /dev/null
+++ b/marimo/_runtime/dataflow/topology.py
@@ -0,0 +1,151 @@
+# Copyright 2024 Marimo. All rights reserved.
+"""Graph topology for cell dependencies.
+
+This module provides the core graph structure for tracking cell relationships.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Protocol
+
+from marimo._runtime.dataflow.types import Edge
+
+if TYPE_CHECKING:
+ from collections.abc import Mapping
+
+ from marimo._ast.cell import CellImpl
+ from marimo._types.ids import CellId_t
+
+
+class GraphTopology(Protocol):
+ """Graph topology protocol.
+
+ This protocol is used to ensure that the graph topology is immutable. All mutations
+ should go through the MutableGraphTopology class.
+ """
+
+ @property
+ def cells(self) -> Mapping[CellId_t, CellImpl]: ...
+
+ @property
+ def children(self) -> Mapping[CellId_t, set[CellId_t]]: ...
+
+ @property
+ def parents(self) -> Mapping[CellId_t, set[CellId_t]]: ...
+
+ def get_path(self, source: CellId_t, dst: CellId_t) -> list[Edge]: ...
+
+ def ancestors(self, cell_id: CellId_t) -> set[CellId_t]: ...
+
+ def descendants(self, cell_id: CellId_t) -> set[CellId_t]: ...
+
+
+@dataclass
+class MutableGraphTopology(GraphTopology):
+ """Pure graph structure: nodes and edges only.
+
+ Responsibilities:
+ - Store cells, parents, children mappings
+ - Provide fast lookups and traversals
+ - No business logic, just data structure
+ """
+
+ # Nodes in the graph
+ _cells: dict[CellId_t, CellImpl] = field(default_factory=dict)
+
+ # Edge (u, v) means v is a child of u, i.e., v has a reference
+ # to something defined in u
+ _children: dict[CellId_t, set[CellId_t]] = field(default_factory=dict)
+
+ # Reversed edges (parent pointers) for convenience
+ _parents: dict[CellId_t, set[CellId_t]] = field(default_factory=dict)
+
+ @property
+ def cells(self) -> Mapping[CellId_t, CellImpl]:
+ return self._cells
+
+ @property
+ def children(self) -> Mapping[CellId_t, set[CellId_t]]:
+ return self._children
+
+ @property
+ def parents(self) -> Mapping[CellId_t, set[CellId_t]]:
+ return self._parents
+
+ def ancestors(self, cell_id: CellId_t) -> set[CellId_t]:
+ """Get all ancestors of a cell."""
+ from marimo._runtime.dataflow import transitive_closure
+
+ return transitive_closure(
+ self, {cell_id}, children=False, inclusive=False
+ )
+
+ def descendants(self, cell_id: CellId_t) -> set[CellId_t]:
+ """Get all descendants of a cell."""
+ from marimo._runtime.dataflow import transitive_closure
+
+ return transitive_closure(self, {cell_id}, inclusive=False)
+
+ def add_node(self, cell_id: CellId_t, cell: CellImpl) -> None:
+ """Add a cell to the graph topology."""
+ assert cell_id not in self.cells, f"Cell {cell_id} already in graph"
+ self._cells[cell_id] = cell
+ self._children[cell_id] = set()
+ self._parents[cell_id] = set()
+
+ def remove_node(self, cell_id: CellId_t) -> None:
+ """Remove a cell from the graph topology.
+
+ Also removes all edges connected to this cell.
+ """
+ if cell_id not in self.cells:
+ raise ValueError(f"Cell {cell_id} not found")
+
+ # Remove from cells
+ del self._cells[cell_id]
+ del self._children[cell_id]
+ del self._parents[cell_id]
+
+ # Remove from other nodes' parent/child lists
+ for elems in self.parents.values():
+ if cell_id in elems:
+ elems.remove(cell_id)
+ for elems in self.children.values():
+ if cell_id in elems:
+ elems.remove(cell_id)
+
+ def add_edge(self, parent: CellId_t, child: CellId_t) -> None:
+ """Add an edge from parent to child."""
+ self.children[parent].add(child)
+ self.parents[child].add(parent)
+
+ def remove_edge(self, parent: CellId_t, child: CellId_t) -> None:
+ """Remove an edge from parent to child."""
+ self.children[parent].discard(child)
+ self.parents[child].discard(parent)
+
+ def get_path(self, source: CellId_t, dst: CellId_t) -> list[Edge]:
+ """Get a path from `source` to `dst`, if any.
+
+ Returns an empty list if source == dst or if no path exists.
+ """
+ if source == dst:
+ return []
+
+ # BFS to find path
+ # deque has O(1) append/pop operation
+ queue: deque[tuple[CellId_t, list[Edge]]] = deque([(source, [])])
+ found = {source} # set has O(1) lookups
+
+ while queue:
+ node, path = queue.popleft() # O(1) operation
+ for cid in self.children[node]:
+ if cid not in found:
+ next_path = path + [(node, cid)]
+ if cid == dst:
+ return next_path
+ found.add(cid)
+ queue.append((cid, next_path))
+ return []
diff --git a/marimo/_runtime/dataflow/types.py b/marimo/_runtime/dataflow/types.py
new file mode 100644
index 0000000000..863ac03113
--- /dev/null
+++ b/marimo/_runtime/dataflow/types.py
@@ -0,0 +1,16 @@
+# Copyright 2024 Marimo. All rights reserved.
+"""Type definitions for dataflow graph."""
+
+from __future__ import annotations
+
+from marimo._types.ids import CellId_t
+
+# Edge is a tuple of (parent, child) cell IDs
+Edge = tuple[CellId_t, CellId_t]
+
+# EdgeWithVar uses a tuple (not a set) for the variables linking the cells
+# as sets are not JSON-serializable (required by static_notebook_template()).
+# The first entry is the source node; the second entry is a tuple of defs from
+# the source read by the destination; and the third entry is the destination
+# node.
+EdgeWithVar = tuple[CellId_t, tuple[str, ...], CellId_t]
diff --git a/marimo/_runtime/packages/module_registry.py b/marimo/_runtime/packages/module_registry.py
index 9c41a301ac..6c24dfe010 100644
--- a/marimo/_runtime/packages/module_registry.py
+++ b/marimo/_runtime/packages/module_registry.py
@@ -4,7 +4,7 @@ from __future__ import annotations
import importlib.util
import sys
-from marimo._runtime.dataflow import DirectedGraph
+from marimo._runtime.dataflow.topology import GraphTopology
from marimo._types.ids import CellId_t
@@ -26,7 +26,7 @@ def _is_module_installed(module_name: str) -> bool:
class ModuleRegistry:
def __init__(
- self, graph: DirectedGraph, excluded_modules: set[str] | None = None
+ self, graph: GraphTopology, excluded_modules: set[str] | None = None
) -> None:
self.graph = graph
# modules that do not have corresponding packages on package index
diff --git a/marimo/_runtime/reload/module_watcher.py b/marimo/_runtime/reload/module_watcher.py
index a8fd412722..d697ccde19 100644
--- a/marimo/_runtime/reload/module_watcher.py
+++ b/marimo/_runtime/reload/module_watcher.py
@@ -205,7 +205,7 @@ def watch_modules(
modname_to_cell_id[modname]
for modname in stale_modules
),
- relatives=dataflow.import_block_relatives,
+ relatives=dataflow.get_import_block_relatives(graph),
)
for cid in stale_cell_ids:
graph.cells[cid].set_stale(stale=True, stream=stream)
diff --git a/marimo/_runtime/runner/cell_runner.py b/marimo/_runtime/runner/cell_runner.py
index 341a00175e..98b0cfd619 100644
--- a/marimo/_runtime/runner/cell_runner.py
+++ b/marimo/_runtime/runner/cell_runner.py
@@ -195,10 +195,13 @@ class Runner:
predicate=lambda cell: cell.stale,
)
)
+
if execution_mode == "autorun":
# in autorun/eager mode, descendants are also run;
cells_to_run = dataflow.transitive_closure(
- graph, cells_to_run, relatives=dataflow.import_block_relatives
+ graph,
+ cells_to_run,
+ relatives=dataflow.get_import_block_relatives(graph),
)
return dataflow.topological_sort(
@@ -643,7 +646,7 @@ class Runner:
blamed_cell = None
try:
(blamed_cell, *_) = self.graph.get_defining_cells(ref)
- except KeyError:
+ except (KeyError, ValueError):
# The reference is not found anywhere else in the graph
# but it might be private
ref, var_cell_id = unmangle_local(ref)
diff --git a/marimo/_runtime/runner/hooks_preparation.py b/marimo/_runtime/runner/hooks_preparation.py
index 9eebb30cdf..dddd28948b 100644
--- a/marimo/_runtime/runner/hooks_preparation.py
+++ b/marimo/_runtime/runner/hooks_preparation.py
@@ -4,7 +4,6 @@ from __future__ import annotations
from typing import Callable
from marimo._runtime import dataflow
-from marimo._runtime.dataflow import import_block_relatives
from marimo._runtime.runner import cell_runner
from marimo._tracer import kernel_tracer
@@ -20,7 +19,7 @@ def _update_stale_statuses(runner: cell_runner.Runner) -> None:
graph,
set(runner.cells_to_run),
inclusive=False,
- relatives=import_block_relatives,
+ relatives=dataflow.get_import_block_relatives(graph),
):
graph.cells[cid].set_stale(stale=True)
diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.py
index 640f9de3f2..5dc1dd4481 100644
--- a/marimo/_runtime/runtime.py
+++ b/marimo/_runtime/runtime.py
@@ -1743,7 +1743,7 @@ class Kernel:
dataflow.transitive_closure(
self.graph,
cells_to_run,
- relatives=dataflow.import_block_relatives,
+ relatives=dataflow.get_import_block_relatives(self.graph),
)
)
if self.module_watcher is not None:
diff --git a/marimo/_save/hash.py b/marimo/_save/hash.py
index 65dcc8f46b..9cf696de01 100644
--- a/marimo/_save/hash.py
+++ b/marimo/_save/hash.py
@@ -22,6 +22,7 @@ from marimo._dependencies.dependencies import DependencyManager
from marimo._plugins.ui._core.ui_element import UIElement
from marimo._runtime.context import ContextNotInitializedError, get_context
from marimo._runtime.dataflow import induced_subgraph
+from marimo._runtime.dataflow.topology import GraphTopology
from marimo._runtime.primitives import (
CLONE_PRIMITIVES,
FN_CACHE_TYPE,
@@ -146,7 +147,7 @@ def hash_function(
def hash_cell_group(
cell_ids: set[CellId_t],
- graph: DirectedGraph,
+ graph: GraphTopology,
hash_type: str = DEFAULT_HASH,
) -> bytes:
hash_alg = hashlib.new(hash_type, usedforsecurity=False)
@@ -162,7 +163,7 @@ def hash_cell_group(
def hash_cell_execution(
- cell_id: CellId_t, graph: DirectedGraph, hash_type: str = DEFAULT_HASH
+ cell_id: CellId_t, graph: GraphTopology, hash_type: str = DEFAULT_HASH
) -> bytes:
ancestors = graph.ancestors(cell_id)
return hash_cell_group(ancestors, graph, hash_type)
diff --git a/tests/_lint/test_validate_graph.py b/tests/_lint/test_validate_graph.py
index 53ef340779..41cb8db68a 100644
--- a/tests/_lint/test_validate_graph.py
+++ b/tests/_lint/test_validate_graph.py
@@ -75,12 +75,16 @@ def test_two_node_cycle() -> None:
graph.register_cell("1", parse_cell("y = x"))
errors = check_for_errors(graph)
assert set(errors.keys()) == set(["0", "1"])
- assert errors["0"] == (
- CycleError(edges_with_vars=(("0", ("x",), "1"), ("1", ("y",), "0"))),
- )
- assert errors["1"] == (
+ # Edge ordering in returned error is not deterministic, so we list both
+ # possible cycles
+ expected_cycle = [
CycleError(edges_with_vars=(("0", ("x",), "1"), ("1", ("y",), "0"))),
- )
+ CycleError(edges_with_vars=(("1", ("y",), "0"), ("0", ("x",), "1"))),
+ ]
+ assert len(errors["0"]) == 1
+ assert len(errors["1"]) == 1
+ assert errors["0"][0] in expected_cycle
+ assert errors["1"][0] in expected_cycle
def test_three_node_cycle() -> None:
diff --git a/tests/_runtime/test_dataflow.py b/tests/_runtime/test_dataflow.py
index d8eab1ffb5..4df98513d8 100644
--- a/tests/_runtime/test_dataflow.py
+++ b/tests/_runtime/test_dataflow.py
@@ -1204,7 +1204,7 @@ def test_import_block_relatives() -> None:
graph.register_cell("3", fourth_cell)
# Test the function
- children = dataflow.import_block_relatives(graph, "0", True)
+ children = dataflow.get_import_block_relatives(graph)("0", True)
# Should include cells that use pd and np
assert "1" in children
@@ -1212,7 +1212,7 @@ def test_import_block_relatives() -> None:
assert "3" not in children
# Test with a non-import block
- children = dataflow.import_block_relatives(graph, "3", True)
+ children = dataflow.get_import_block_relatives(graph)("3", True)
# Should just return normal children
assert children == graph.children["3"]
diff --git a/tests/_runtime/test_edges.py b/tests/_runtime/test_edges.py
new file mode 100644
index 0000000000..2dd8dd897f
--- /dev/null
+++ b/tests/_runtime/test_edges.py
@@ -0,0 +1,564 @@
+# Copyright 2024 Marimo. All rights reserved.
+from __future__ import annotations
+
+from functools import partial
+
+import pytest
+
+from marimo._ast import compiler
+from marimo._dependencies.dependencies import DependencyManager
+from marimo._runtime.dataflow.definitions import DefinitionRegistry
+from marimo._runtime.dataflow.edges import EdgeComputer
+from marimo._runtime.dataflow.topology import MutableGraphTopology
+
+parse_cell = partial(compiler.compile_cell, cell_id="0")
+
+HAS_SQL = DependencyManager.duckdb.has() and DependencyManager.polars.has()
+
+
+class TestEdgeComputer:
+ """Tests for EdgeComputer class."""
+
+ def setup_method(self) -> None:
+ """Set up a fresh topology and definitions for each test."""
+ self.topology = MutableGraphTopology()
+ self.definitions = DefinitionRegistry()
+ self.edge_computer = EdgeComputer(self.topology, self.definitions)
+
+ def test_get_referring_cells_python_no_refs(self) -> None:
+ """Test getting referring cells when no cells reference the variable."""
+ cell1 = parse_cell("x = 1")
+ self.topology.add_node("cell_1", cell1)
+
+ referring = self.edge_computer.get_referring_cells(
+ "x", language="python"
+ )
+
+ assert referring == set()
+
+ def test_get_referring_cells_python_single_ref(self) -> None:
+ """Test getting referring cells with a single reference."""
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+
+ referring = self.edge_computer.get_referring_cells(
+ "x", language="python"
+ )
+
+ assert referring == {"cell_2"}
+
+ def test_get_referring_cells_python_multiple_refs(self) -> None:
+ """Test getting referring cells with multiple references."""
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+ cell3 = parse_cell("z = x * 2")
+ cell4 = parse_cell("w = x + y")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+ self.topology.add_node("cell_3", cell3)
+ self.topology.add_node("cell_4", cell4)
+
+ referring = self.edge_computer.get_referring_cells(
+ "x", language="python"
+ )
+
+ assert referring == {"cell_2", "cell_3", "cell_4"}
+
+ def test_get_referring_cells_python_no_self_reference(self) -> None:
+ """Test that a cell doesn't include itself in referring cells."""
+ cell1 = parse_cell("x = x + 1")
+
+ self.topology.add_node("cell_1", cell1)
+
+ referring = self.edge_computer.get_referring_cells(
+ "x", language="python"
+ )
+
+ # cell_1 both defines and references x, but should be in the result
+ assert referring == {"cell_1"}
+
+ def test_compute_edges_simple_dependency(self) -> None:
+ """Test computing edges for a simple dependency."""
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+
+ # Register cell1's definitions
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+
+ # Compute edges for cell2
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_2", cell2
+ )
+
+ assert parents == {"cell_1"}
+ assert children == set()
+
+ def test_compute_edges_multiple_parents(self) -> None:
+ """Test computing edges with multiple parent dependencies."""
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = 2")
+ cell3 = parse_cell("z = x + y")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+ self.topology.add_node("cell_3", cell3)
+
+ # Register definitions
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+ for name, var_data in cell2.variable_data.items():
+ self.definitions.register_definition("cell_2", name, var_data)
+
+ # Compute edges for cell3
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_3", cell3
+ )
+
+ assert parents == {"cell_1", "cell_2"}
+ assert children == set()
+
+ def test_compute_edges_with_children(self) -> None:
+ """Test computing edges when a cell has both parents and children."""
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+ cell3 = parse_cell("z = y")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+ self.topology.add_node("cell_3", cell3)
+
+ # Register definitions for cell1
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+
+ # Register definitions for cell2
+ for name, var_data in cell2.variable_data.items():
+ self.definitions.register_definition("cell_2", name, var_data)
+
+ # Compute edges for cell2 (middle cell)
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_2", cell2
+ )
+
+ assert parents == {"cell_1"}
+ assert children == {"cell_3"}
+
+ def test_compute_edges_no_dependencies(self) -> None:
+ """Test computing edges for an independent cell."""
+ cell1 = parse_cell("x = 1")
+
+ self.topology.add_node("cell_1", cell1)
+
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_1", cell1
+ )
+
+ assert parents == set()
+ assert children == set()
+
+ def test_compute_edges_with_deleted_refs(self) -> None:
+ """Test computing edges when a cell deletes a variable."""
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+ cell3 = parse_cell("del x")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+ self.topology.add_node("cell_3", cell3)
+
+ # Register definitions
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+
+ # Compute edges for cell3 (which deletes x)
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_3", cell3
+ )
+
+ # cell3 should become a child of cells that reference x (cell2)
+ assert parents == {"cell_1", "cell_2"}
+ assert children == set()
+
+ def test_compute_edges_multiple_deleted_refs_create_cycle(self) -> None:
+ """Test that multiple cells deleting the same variable create cycles."""
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("del x")
+ cell3 = parse_cell("del x")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+ self.topology.add_node("cell_3", cell3)
+
+ # Register definitions
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+
+ # Compute edges for cell2
+ parents2, children2 = self.edge_computer.compute_edges_for_cell(
+ "cell_2", cell2
+ )
+
+ # Compute edges for cell3
+ parents3, children3 = self.edge_computer.compute_edges_for_cell(
+ "cell_3", cell3
+ )
+
+ # Both cells should have cell1 as parent
+ assert "cell_1" in parents2
+ assert "cell_1" in parents3
+
+ # cell2 and cell3 should be children of each other (cycle)
+ assert "cell_3" in children2
+ assert "cell_2" in parents3
+
+ def test_compute_edges_cell_references_its_own_def(self) -> None:
+ """Test computing edges when a cell references its own definition."""
+ cell1 = parse_cell("x = x + 1 if 'x' in globals() else 0")
+
+ self.topology.add_node("cell_1", cell1)
+
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_1", cell1
+ )
+
+ # A cell doesn't create an edge to itself
+ assert parents == set()
+ assert children == set()
+
+ def test_compute_edges_diamond_dependency(self) -> None:
+ """Test computing edges in a diamond-shaped dependency graph."""
+ cell1 = parse_cell("a = 1")
+ cell2 = parse_cell("b = a")
+ cell3 = parse_cell("c = a")
+ cell4 = parse_cell("d = b + c")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+ self.topology.add_node("cell_3", cell3)
+ self.topology.add_node("cell_4", cell4)
+
+ # Register definitions
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+ for name, var_data in cell2.variable_data.items():
+ self.definitions.register_definition("cell_2", name, var_data)
+ for name, var_data in cell3.variable_data.items():
+ self.definitions.register_definition("cell_3", name, var_data)
+
+ # Check cell1's children (only direct children, not grandchildren)
+ parents1, children1 = self.edge_computer.compute_edges_for_cell(
+ "cell_1", cell1
+ )
+ assert children1 == {"cell_2", "cell_3"}
+
+ # Check cell4's parents (it references b and c, not a directly)
+ parents4, children4 = self.edge_computer.compute_edges_for_cell(
+ "cell_4", cell4
+ )
+ assert parents4 == {"cell_2", "cell_3"}
+
+ def test_compute_edges_with_undefined_reference(self) -> None:
+ """Test computing edges when a cell references an undefined variable."""
+ cell1 = parse_cell("y = x") # x is not defined
+
+ self.topology.add_node("cell_1", cell1)
+
+ # Should not raise an error, just return empty parents
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_1", cell1
+ )
+
+ assert parents == set()
+ assert children == set()
+
+ def test_compute_edges_with_function_closure(self) -> None:
+ """Test computing edges with function closures."""
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("def foo():\n return x")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+
+ # Register definitions
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+
+ # Compute edges for cell2
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_2", cell2
+ )
+
+ assert parents == {"cell_1"}
+ assert children == set()
+
+ def test_compute_edges_with_class_definition(self) -> None:
+ """Test computing edges with class definitions."""
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("class MyClass:\n value = x")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+
+ # Register definitions
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+
+ # Compute edges for cell2
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_2", cell2
+ )
+
+ assert parents == {"cell_1"}
+ assert children == set()
+
+ def test_compute_edges_chain_of_dependencies(self) -> None:
+ """Test a long chain of dependencies."""
+ cells = []
+ for i in range(5):
+ if i == 0:
+ cell = parse_cell(f"x{i} = {i}")
+ else:
+ cell = parse_cell(f"x{i} = x{i - 1} + 1")
+ self.topology.add_node(f"cell_{i}", cell)
+ cells.append((f"cell_{i}", cell))
+
+ # Register definitions
+ for cell_id, cell in cells[:-1]:
+ for name, var_data in cell.variable_data.items():
+ self.definitions.register_definition(cell_id, name, var_data)
+
+ # Check last cell's parents
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_4", cells[-1][1]
+ )
+
+ assert parents == {"cell_3"}
+ assert children == set()
+
+ @pytest.mark.skipif(not HAS_SQL, reason="requires duckdb and polars")
+ def test_get_referring_cells_sql_basic(self) -> None:
+ """Test getting referring cells for SQL variables."""
+ # SQL cell that creates a table
+ cell1 = compiler.compile_cell(
+ "import duckdb; import polars as pl; df = pl.DataFrame({'a': [1, 2, 3]})",
+ cell_id="0",
+ )
+ cell2 = compiler.compile_cell("df.select(pl.col('a'))", cell_id="1")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+
+ referring = self.edge_computer.get_referring_cells(
+ "df", language="python"
+ )
+
+ assert referring == {"cell_2"}
+
+ @pytest.mark.skipif(not HAS_SQL, reason="requires duckdb and polars")
+ def test_compute_edges_sql_to_python(self) -> None:
+ """Test that SQL table definitions don't create edges to Python refs."""
+ # This tests the language isolation: SQL defs don't leak to Python
+ cell1 = compiler.compile_cell(
+ "df = mo.sql('CREATE TABLE my_table AS SELECT 1 as x')",
+ cell_id="0",
+ )
+ cell2 = compiler.compile_cell("print(my_table)", cell_id="1")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+
+ # Register SQL table definition
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+
+ # Compute edges for cell2 (Python cell)
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_2", cell2
+ )
+
+ # SQL definitions should not create edges to Python cells
+ # (unless explicitly exported)
+ # Since my_table is SQL, it won't be a parent
+ # However, the compiler might not recognize it as SQL without execution
+ # So this test verifies the edge computation logic
+
+ def test_compute_edges_multiple_definitions_of_same_var(self) -> None:
+ """Test computing edges when multiple cells define the same variable."""
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("x = 2")
+ cell3 = parse_cell("y = x")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+ self.topology.add_node("cell_3", cell3)
+
+ # Register both definitions of x
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+ for name, var_data in cell2.variable_data.items():
+ self.definitions.register_definition("cell_2", name, var_data)
+
+ # Compute edges for cell3
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_3", cell3
+ )
+
+ # cell3 should depend on both cells that define x
+ assert parents == {"cell_1", "cell_2"}
+
+ def test_compute_edges_complex_scenario(self) -> None:
+ """Test a complex scenario with multiple patterns."""
+ # Create a complex graph:
+ # cell1: defines a
+ # cell2: defines b = a
+ # cell3: defines c = a + b
+ # cell4: uses c, deletes a
+ # cell5: defines d = b
+
+ cell1 = parse_cell("a = 1")
+ cell2 = parse_cell("b = a")
+ cell3 = parse_cell("c = a + b")
+ cell4 = parse_cell("result = c; del a")
+ cell5 = parse_cell("d = b")
+
+ cells = [
+ ("cell_1", cell1),
+ ("cell_2", cell2),
+ ("cell_3", cell3),
+ ("cell_4", cell4),
+ ("cell_5", cell5),
+ ]
+
+ for cell_id, cell in cells:
+ self.topology.add_node(cell_id, cell)
+
+ # Register definitions for cells 1-3
+ for cell_id, cell in cells[:3]:
+ for name, var_data in cell.variable_data.items():
+ self.definitions.register_definition(cell_id, name, var_data)
+
+ # Compute edges for cell4
+ parents4, children4 = self.edge_computer.compute_edges_for_cell(
+ "cell_4", cell4
+ )
+
+ # cell4 should:
+ # - depend on cell3 (for c)
+ # - depend on cell1 (for del a)
+ # - depend on cell2 (because it references a, which cell4 deletes)
+ assert "cell_3" in parents4 # uses c
+ assert "cell_1" in parents4 # deletes a
+ assert "cell_2" in parents4 # cell2 references a
+
+ def test_is_valid_cell_reference_missing_cell(self) -> None:
+ """Test validation when a cell reference points to a non-existent cell."""
+ # The _is_valid_cell_reference method logs an error when a cell is not found
+ result = self.edge_computer._is_valid_cell_reference(
+ "nonexistent", "some_var"
+ )
+
+ assert result is False
+
+ def test_is_valid_cell_reference_valid_cell(self) -> None:
+ """Test validation when a cell reference is valid."""
+ cell1 = parse_cell("x = 1")
+ self.topology.add_node("cell_1", cell1)
+
+ result = self.edge_computer._is_valid_cell_reference("cell_1", "x")
+
+ assert result is True
+
+ def test_compute_edges_with_imports(self) -> None:
+ """Test computing edges with import statements."""
+ cell1 = parse_cell("import math")
+ cell2 = parse_cell("x = math.pi")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+
+ # Register definitions
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+
+ # Compute edges for cell2
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_2", cell2
+ )
+
+ assert parents == {"cell_1"}
+
+ def test_compute_edges_with_from_import(self) -> None:
+ """Test computing edges with from-import statements."""
+ cell1 = parse_cell("from math import pi")
+ cell2 = parse_cell("x = pi")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+
+ # Register definitions
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+
+ # Compute edges for cell2
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_2", cell2
+ )
+
+ assert parents == {"cell_1"}
+
+ def test_compute_edges_list_comprehension(self) -> None:
+ """Test computing edges with list comprehensions."""
+ cell1 = parse_cell("numbers = [1, 2, 3]")
+ cell2 = parse_cell("squared = [x**2 for x in numbers]")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+
+ # Register definitions
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+
+ # Compute edges for cell2
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ "cell_2", cell2
+ )
+
+ assert parents == {"cell_1"}
+
+ def test_compute_edges_with_multiple_assignments(self) -> None:
+ """Test computing edges with multiple assignments."""
+ cell1 = parse_cell("x = y = z = 1")
+ cell2 = parse_cell("a = x")
+ cell3 = parse_cell("b = y")
+ cell4 = parse_cell("c = z")
+
+ self.topology.add_node("cell_1", cell1)
+ self.topology.add_node("cell_2", cell2)
+ self.topology.add_node("cell_3", cell3)
+ self.topology.add_node("cell_4", cell4)
+
+ # Register definitions
+ for name, var_data in cell1.variable_data.items():
+ self.definitions.register_definition("cell_1", name, var_data)
+
+ # All cells should depend on cell1
+ for cell_id in ["cell_2", "cell_3", "cell_4"]:
+ cell = self.topology.cells[cell_id]
+ parents, children = self.edge_computer.compute_edges_for_cell(
+ cell_id, cell
+ )
+ assert parents == {"cell_1"}
+
+ # Check that cell1 has all three as children
+ parents1, children1 = self.edge_computer.compute_edges_for_cell(
+ "cell_1", cell1
+ )
+ assert children1 == {"cell_2", "cell_3", "cell_4"}
diff --git a/tests/_runtime/test_topology.py b/tests/_runtime/test_topology.py
new file mode 100644
index 0000000000..e620381a27
--- /dev/null
+++ b/tests/_runtime/test_topology.py
@@ -0,0 +1,429 @@
+# Copyright 2024 Marimo. All rights reserved.
+from __future__ import annotations
+
+from functools import partial
+
+import pytest
+
+from marimo._ast import compiler
+from marimo._runtime.dataflow.topology import MutableGraphTopology
+from marimo._types.ids import CellId_t
+
+parse_cell = partial(compiler.compile_cell, cell_id=CellId_t("0"))
+
+
+class TestMutableGraphTopology:
+ """Tests for MutableGraphTopology class."""
+
+ def test_empty_graph(self) -> None:
+ """Test that an empty graph has no cells, parents, or children."""
+ graph = MutableGraphTopology()
+ assert len(graph.cells) == 0
+ assert len(graph.children) == 0
+ assert len(graph.parents) == 0
+
+ def test_add_single_node(self) -> None:
+ """Test adding a single node to the graph."""
+ graph = MutableGraphTopology()
+ cell = parse_cell("x = 1")
+
+ graph.add_node("cell_1", cell)
+
+ assert "cell_1" in graph.cells
+ assert graph.cells["cell_1"] == cell
+ assert graph.children["cell_1"] == set()
+ assert graph.parents["cell_1"] == set()
+
+ def test_add_duplicate_node_raises_error(self) -> None:
+ """Test that adding a duplicate node raises an assertion error."""
+ graph = MutableGraphTopology()
+ cell = parse_cell("x = 1")
+
+ graph.add_node("cell_1", cell)
+
+ with pytest.raises(
+ AssertionError, match="Cell cell_1 already in graph"
+ ):
+ graph.add_node("cell_1", cell)
+
+ def test_add_multiple_nodes(self) -> None:
+ """Test adding multiple nodes to the graph."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = 2")
+ cell3 = parse_cell("z = 3")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_node("cell_3", cell3)
+
+ assert len(graph.cells) == 3
+ assert "cell_1" in graph.cells
+ assert "cell_2" in graph.cells
+ assert "cell_3" in graph.cells
+
+ def test_remove_node(self) -> None:
+ """Test removing a node from the graph."""
+ graph = MutableGraphTopology()
+ cell = parse_cell("x = 1")
+
+ graph.add_node("cell_1", cell)
+ graph.remove_node("cell_1")
+
+ assert "cell_1" not in graph.cells
+ assert "cell_1" not in graph.children
+ assert "cell_1" not in graph.parents
+
+ def test_remove_nonexistent_node_raises_error(self) -> None:
+ """Test that removing a non-existent node raises ValueError."""
+ graph = MutableGraphTopology()
+
+ with pytest.raises(ValueError, match="Cell cell_1 not found"):
+ graph.remove_node("cell_1")
+
+ def test_remove_node_with_edges(self) -> None:
+ """Test that removing a node also removes its edges."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+ cell3 = parse_cell("z = y")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_node("cell_3", cell3)
+ graph.add_edge("cell_1", "cell_2")
+ graph.add_edge("cell_2", "cell_3")
+
+ # Remove middle node
+ graph.remove_node("cell_2")
+
+ # Check that cell_2 is removed from children and parents
+ assert "cell_2" not in graph.cells
+ assert "cell_2" not in graph.children["cell_1"]
+ assert "cell_2" not in graph.parents["cell_3"]
+
+ def test_add_edge(self) -> None:
+ """Test adding an edge between two nodes."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_edge("cell_1", "cell_2")
+
+ assert "cell_2" in graph.children["cell_1"]
+ assert "cell_1" in graph.parents["cell_2"]
+
+ def test_add_multiple_edges(self) -> None:
+ """Test adding multiple edges in the graph."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+ cell3 = parse_cell("z = x + y")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_node("cell_3", cell3)
+ graph.add_edge("cell_1", "cell_2")
+ graph.add_edge("cell_1", "cell_3")
+ graph.add_edge("cell_2", "cell_3")
+
+ # cell_1 has two children
+ assert graph.children["cell_1"] == {"cell_2", "cell_3"}
+ # cell_2 has one child
+ assert graph.children["cell_2"] == {"cell_3"}
+ # cell_3 has two parents
+ assert graph.parents["cell_3"] == {"cell_1", "cell_2"}
+
+ def test_remove_edge(self) -> None:
+ """Test removing an edge between two nodes."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_edge("cell_1", "cell_2")
+ graph.remove_edge("cell_1", "cell_2")
+
+ assert "cell_2" not in graph.children["cell_1"]
+ assert "cell_1" not in graph.parents["cell_2"]
+
+ def test_remove_nonexistent_edge(self) -> None:
+ """Test that removing a non-existent edge doesn't raise error."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = 2")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+
+ # Should not raise an error (using discard)
+ graph.remove_edge("cell_1", "cell_2")
+
+ assert "cell_2" not in graph.children["cell_1"]
+ assert "cell_1" not in graph.parents["cell_2"]
+
+ def test_get_path_direct_edge(self) -> None:
+ """Test getting a path between two directly connected nodes."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_edge("cell_1", "cell_2")
+
+ path = graph.get_path("cell_1", "cell_2")
+
+ assert path == [("cell_1", "cell_2")]
+
+ def test_get_path_indirect(self) -> None:
+ """Test getting a path through intermediate nodes."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+ cell3 = parse_cell("z = y")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_node("cell_3", cell3)
+ graph.add_edge("cell_1", "cell_2")
+ graph.add_edge("cell_2", "cell_3")
+
+ path = graph.get_path("cell_1", "cell_3")
+
+ assert path == [("cell_1", "cell_2"), ("cell_2", "cell_3")]
+
+ def test_get_path_no_path(self) -> None:
+ """Test getting a path when no path exists."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = 2")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+
+ path = graph.get_path("cell_1", "cell_2")
+
+ assert path == []
+
+ def test_get_path_same_node(self) -> None:
+ """Test getting a path from a node to itself."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+
+ graph.add_node("cell_1", cell1)
+
+ path = graph.get_path("cell_1", "cell_1")
+
+ assert path == []
+
+ def test_get_path_complex_graph(self) -> None:
+ """Test getting a path in a more complex graph with multiple paths."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("a = 1")
+ cell2 = parse_cell("b = a")
+ cell3 = parse_cell("c = a")
+ cell4 = parse_cell("d = b + c")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_node("cell_3", cell3)
+ graph.add_node("cell_4", cell4)
+ graph.add_edge("cell_1", "cell_2")
+ graph.add_edge("cell_1", "cell_3")
+ graph.add_edge("cell_2", "cell_4")
+ graph.add_edge("cell_3", "cell_4")
+
+ # BFS should find one of the shortest paths
+ path = graph.get_path("cell_1", "cell_4")
+
+ # Should be length 2 (one of: cell_1->cell_2->cell_4 or cell_1->cell_3->cell_4)
+ assert len(path) == 2
+ assert path[0][0] == "cell_1"
+ assert path[1][1] == "cell_4"
+
+ def test_ancestors_empty(self) -> None:
+ """Test ancestors of a node with no parents."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+
+ graph.add_node("cell_1", cell1)
+
+ ancestors = graph.ancestors("cell_1")
+
+ assert ancestors == set()
+
+ def test_ancestors_single_parent(self) -> None:
+ """Test ancestors with a single parent."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_edge("cell_1", "cell_2")
+
+ ancestors = graph.ancestors("cell_2")
+
+ assert ancestors == {"cell_1"}
+
+ def test_ancestors_multiple_generations(self) -> None:
+ """Test ancestors across multiple generations."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+ cell3 = parse_cell("z = y")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_node("cell_3", cell3)
+ graph.add_edge("cell_1", "cell_2")
+ graph.add_edge("cell_2", "cell_3")
+
+ ancestors = graph.ancestors("cell_3")
+
+ assert ancestors == {"cell_1", "cell_2"}
+
+ def test_ancestors_diamond_shape(self) -> None:
+ """Test ancestors in a diamond-shaped graph."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("a = 1")
+ cell2 = parse_cell("b = a")
+ cell3 = parse_cell("c = a")
+ cell4 = parse_cell("d = b + c")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_node("cell_3", cell3)
+ graph.add_node("cell_4", cell4)
+ graph.add_edge("cell_1", "cell_2")
+ graph.add_edge("cell_1", "cell_3")
+ graph.add_edge("cell_2", "cell_4")
+ graph.add_edge("cell_3", "cell_4")
+
+ ancestors = graph.ancestors("cell_4")
+
+ assert ancestors == {"cell_1", "cell_2", "cell_3"}
+
+ def test_descendants_empty(self) -> None:
+ """Test descendants of a node with no children."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+
+ graph.add_node("cell_1", cell1)
+
+ descendants = graph.descendants("cell_1")
+
+ assert descendants == set()
+
+ def test_descendants_single_child(self) -> None:
+ """Test descendants with a single child."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_edge("cell_1", "cell_2")
+
+ descendants = graph.descendants("cell_1")
+
+ assert descendants == {"cell_2"}
+
+ def test_descendants_multiple_generations(self) -> None:
+ """Test descendants across multiple generations."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+ cell2 = parse_cell("y = x")
+ cell3 = parse_cell("z = y")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_node("cell_3", cell3)
+ graph.add_edge("cell_1", "cell_2")
+ graph.add_edge("cell_2", "cell_3")
+
+ descendants = graph.descendants("cell_1")
+
+ assert descendants == {"cell_2", "cell_3"}
+
+ def test_descendants_diamond_shape(self) -> None:
+ """Test descendants in a diamond-shaped graph."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("a = 1")
+ cell2 = parse_cell("b = a")
+ cell3 = parse_cell("c = a")
+ cell4 = parse_cell("d = b + c")
+
+ graph.add_node("cell_1", cell1)
+ graph.add_node("cell_2", cell2)
+ graph.add_node("cell_3", cell3)
+ graph.add_node("cell_4", cell4)
+ graph.add_edge("cell_1", "cell_2")
+ graph.add_edge("cell_1", "cell_3")
+ graph.add_edge("cell_2", "cell_4")
+ graph.add_edge("cell_3", "cell_4")
+
+ descendants = graph.descendants("cell_1")
+
+ assert descendants == {"cell_2", "cell_3", "cell_4"}
+
+ def test_properties_are_readonly(self) -> None:
+ """Test that the properties return Mapping (read-only) types."""
+ graph = MutableGraphTopology()
+ cell1 = parse_cell("x = 1")
+
+ graph.add_node("cell_1", cell1)
+
+ # The properties should return Mapping types
+ from collections.abc import Mapping
+
+ assert isinstance(graph.cells, Mapping)
+ assert isinstance(graph.children, Mapping)
+ assert isinstance(graph.parents, Mapping)
+
+ def test_complex_graph_operations(self) -> None:
+ """Test a complex sequence of operations on the graph."""
+ graph = MutableGraphTopology()
+
+ # Build a graph: 1 -> 2 -> 3 -> 4
+ # \-> 5 -> 6
+ cells = {f"cell_{i}": parse_cell(f"x{i} = {i}") for i in range(1, 7)}
+
+ for cell_id, cell in cells.items():
+ graph.add_node(cell_id, cell)
+
+ graph.add_edge("cell_1", "cell_2")
+ graph.add_edge("cell_2", "cell_3")
+ graph.add_edge("cell_3", "cell_4")
+ graph.add_edge("cell_2", "cell_5")
+ graph.add_edge("cell_5", "cell_6")
+
+ # Test various operations
+ assert graph.descendants("cell_1") == {
+ "cell_2",
+ "cell_3",
+ "cell_4",
+ "cell_5",
+ "cell_6",
+ }
+ assert graph.descendants("cell_2") == {
+ "cell_3",
+ "cell_4",
+ "cell_5",
+ "cell_6",
+ }
+ assert graph.ancestors("cell_4") == {"cell_1", "cell_2", "cell_3"}
+ assert graph.ancestors("cell_6") == {"cell_1", "cell_2", "cell_5"}
+
+ # Remove an edge and test again
+ graph.remove_edge("cell_2", "cell_5")
+ assert graph.descendants("cell_2") == {"cell_3", "cell_4"}
+
+ # Remove a node and test
+ graph.remove_node("cell_3")
+ assert graph.descendants("cell_1") == {"cell_2"}
+ assert "cell_3" not in graph.cells