A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
refactor: split up DirectedGraph into smaller components (#7527)
Myles Scolnick004c787a7ea2
diff --git a/marimo/_ast/toplevel.py b/marimo/_ast/toplevel.pyindex c3bc5a4a5b..228327809c 100644--- a/marimo/_ast/toplevel.py+++ b/marimo/_ast/toplevel.py@@ -17,7 +17,8 @@ from marimo._ast.names import ()from marimo._ast.variables import BUILTINSfrom marimo._ast.visitor import Name, VariableData-from marimo._runtime.dataflow import DirectedGraph, topological_sort+from marimo._runtime.dataflow import topological_sort+from marimo._runtime.dataflow.topology import GraphTopologyfrom marimo._types.ids import CellId_tif TYPE_CHECKING:@@ -379,7 +380,7 @@ class TopLevelExtraction:@classmethoddef from_graph(cls,- graph: DirectedGraph,+ graph: GraphTopology,cell: CellImpl | None = None,) -> TopLevelExtraction:if cell:diff --git a/marimo/_lint/validate_graph.py b/marimo/_lint/validate_graph.pyindex 992526ea77..8e798a0b2b 100644--- a/marimo/_lint/validate_graph.py+++ b/marimo/_lint/validate_graph.py@@ -12,6 +12,7 @@ from marimo._messaging.errors import (SetupRootError,)from marimo._runtime.dataflow import DirectedGraph+from marimo._runtime.dataflow.topology import GraphTopologyfrom marimo._types.ids import CellId_t@@ -68,7 +69,7 @@ def check_for_multiple_definitions(def check_for_invalid_root(- graph: DirectedGraph,+ graph: GraphTopology,) -> dict[CellId_t, list[SetupRootError]]:"""Setup cell cannot have parents."""errors: dict[CellId_t, list[SetupRootError]] = defaultdict(list)diff --git a/marimo/_runtime/dataflow.py b/marimo/_runtime/dataflow.pydeleted file mode 100644index 4148c7a24b..0000000000--- a/marimo/_runtime/dataflow.py+++ /dev/null@@ -1,1043 +0,0 @@-# Copyright 2024 Marimo. All rights reserved.-from __future__ import annotations--import threading-from collections import deque-from dataclasses import dataclass, field-from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, cast--from marimo import _loggers-from marimo._ast.cell import CellImpl-from marimo._ast.compiler import code_key-from marimo._ast.sql_visitor import SQLRef, SQLTypes-from marimo._ast.variables import is_mangled_local-from marimo._ast.visitor import ImportData, Name, VariableData-from marimo._runtime.executor import (- ExecutionConfig,- get_executor,-)-from marimo._types.ids import CellId_t--if TYPE_CHECKING:- from collections.abc import Collection---Edge = tuple[CellId_t, CellId_t]-# EdgeWithVar uses a list rather than a set for the variables linking the cells-# as sets are not JSON-serializable (required by static_notebook_template()).-# The first entry is the source node; the second entry is a list of defs from-# the source read by the destination; and the third entry is the destination-# node.-EdgeWithVar = tuple[CellId_t, tuple[str, ...], CellId_t]--LOGGER = _loggers.marimo_logger()---# TODO(akshayka): Add method disable_cell, enable_cell which handle-# state transitions on cells-@dataclass(frozen=True)-class DirectedGraph:- # Nodes in the graph- cells: dict[CellId_t, CellImpl] = field(default_factory=dict)-- # Edge (u, v) means v is a child of u, i.e., v has a reference- # to something defined in u- children: dict[CellId_t, set[CellId_t]] = field(default_factory=dict)-- # Reversed edges (parent pointers) for convenience- parents: dict[CellId_t, set[CellId_t]] = field(default_factory=dict)-- # Cells that define the same name- #- # siblings[cell_id] is a set of cell ids, one for each cell that shares a- # definition with cell_id.- #- # If this dict is non-empty, then the marimo program contains multiply- # defined names (and is therefore in an error state)- siblings: dict[CellId_t, set[CellId_t]] = field(default_factory=dict)-- # A mapping from defs to the cells that define them- definitions: dict[Name, set[CellId_t]] = field(default_factory=dict)- typed_definitions: dict[tuple[Name, str], set[CellId_t]] = field(- default_factory=dict- )- definition_types: dict[Name, set[str]] = field(default_factory=dict)-- # The set of cycles in the graph- cycles: set[tuple[Edge, ...]] = field(default_factory=set)-- # This lock must be acquired during methods that mutate the graph; it's- # only needed because a graph is shared between the kernel and the code- # completion service. It should almost always be uncontended.- lock: threading.Lock = field(default_factory=threading.Lock)-- def is_cell_cached(self, cell_id: CellId_t, code: str) -> bool:- """Whether a cell with id `cell_id` and code `code` is in the graph."""- return (- cell_id in self.cells and code_key(code) == self.cells[cell_id].key- )-- # TODO: language type?- def get_defining_cells(self, name: Name) -> set[CellId_t]:- """Get all cells that define name.-- This is a singleton for well-formed graphs.- """- return self.definitions[name]-- def get_referring_cells(- self, name: Name, language: Literal["python", "sql"]- ) -> set[CellId_t]:- """Get all cells that have a ref to `name`.-- The variable can be either a Python variable or a SQL variable (table).- SQL variables don't leak to Python cells, but Python variables do leak- to SQL.-- Only does a local analysis of refs, without taking into consideration- whether refs are defined by other cells.- """- if language == "sql":- # For SQL, only return SQL cells that reference the name- cells = set()- for cid, cell in self.cells.items():- if cell.language != "sql":- continue-- for ref in cell.refs:- # Direct reference match- if ref == name:- cells.add(cid)- break-- sql_ref = cell.sql_refs.get(ref)-- kind: SQLTypes = "any"- if name in cell.variable_data:- variable_data = cell.variable_data[name][-1]- kind = cast(SQLTypes, variable_data.kind)-- # Hierarchical reference match- if sql_ref and sql_ref.matches_hierarchical_ref(- name, ref, kind- ):- cells.add(cid)- break-- return cells- else:- # For Python, return all cells that reference the name- return {- cid for cid, cell in self.cells.items() if name in cell.refs- }-- def _find_sql_hierarchical_matches(- self, sql_ref: SQLRef- ) -> list[tuple[set[CellId_t], Name]]:- """- This method searches through all definitions in the graph to find cells- that define the individual components (table, schema, or catalog) of the- hierarchical reference.-- For example, given a reference "my_schema.my_table", this method will:- - Look for cells that define a table/view named "my_table"- - Look for cells that define a catalog named "my_schema"- (when the reference has at least 2 parts)-- Args:- sql_ref: A hierarchical SQL reference (e.g., "schema.table",- "catalog.schema.table") to find matching definitions for.-- Returns:- A list of tuples containing:- - A set of cell IDs that define components of the hierarchical reference- - The definition of the name that was found (e.g., "schema.table" -> "table")- """- matching_cell_ids_list = []-- for (def_name, kind), cell_ids in self.typed_definitions.items():- # Match table/view definitions- if sql_ref.contains_hierarchical_ref(def_name, kind):- matching_cell_ids_list.append((cell_ids, def_name))-- return matching_cell_ids_list-- def _is_valid_cell_reference(- self, cell_id: CellId_t, variable_name: Name- ) -> bool:- """Check if a cell reference is valid and log errors if not."""- if cell_id not in self.cells:- LOGGER.error(- "Variable %s is defined in cell %s, but is not in the graph",- variable_name,- cell_id,- )- return False- return True-- def _resolve_variable_name(- self,- name: Name,- other_cell: CellImpl,- sql_ref: Optional[SQLRef],- sql_matches: list[tuple[set[CellId_t], Name]],- ) -> Name:- """- Resolve the variable name to use when checking if it exists in another cell.-- For regular (non-SQL) references, returns the original name unchanged.- For SQL hierarchical references, finds the variable name from sql_matches that- is actually defined in the other_cell.-- Example:- cell_1: CREATE SCHEMA schema_name- cell_2: CREATE TABLE schema_name.table_name- cell_3: FROM schema_name.table_name SELECT *-- When cell_3 references "schema_name.table_name":- - For cell_1: returns "schema_name" (what cell_1 actually defines)- - For cell_2: returns "table_name" (what cell_2 actually defines)- """- if not sql_ref or name in other_cell.variable_data:- return name-- # For SQL hierarchical references, find the matching variable name- for _, matching_variable_name in sql_matches:- if matching_variable_name in other_cell.variable_data:- return matching_variable_name-- return name-- def get_path(self, source: CellId_t, dst: CellId_t) -> list[Edge]:- """Get a path from `source` to `dst`, if any."""- if source == dst:- return []-- # deque has O(1) append/pop operation- queue: deque[tuple[CellId_t, list[Edge]]] = deque([(source, [])])- found = {source} # set has O(1) lookups-- while queue:- node, path = queue.popleft() # O(1) operation- for cid in self.children[node]:- if cid not in found:- next_path = path + [(node, cid)]- if cid == dst:- return next_path- found.add(cid)- queue.append((cid, next_path))- return []-- def register_cell(self, cell_id: CellId_t, cell: CellImpl) -> None:- """Add a cell to the graph.-- Mutates the graph, acquiring `self.lock`.-- Requires that `cell_id` is not already in the graph.- """- LOGGER.debug("Acquiring graph lock to register cell %s", cell_id)- with self.lock:- LOGGER.debug("Acquired graph lock.")- assert cell_id not in self.cells- self.cells[cell_id] = cell- # Children are the set of cells that refer to a name defined in- # `cell`- children: set[CellId_t] = set()- # Cells that define the same name as this one- siblings: set[CellId_t] = set()- # Parents are the set of cells that define a name referred to by- # `cell`- parents: set[CellId_t] = set()-- # Populate children, siblings, and parents- self.children[cell_id] = children- self.siblings[cell_id] = siblings- self.parents[cell_id] = parents-- # First, we process the variables that this cell defines. Any cell- # that refers to a defined variable becomes a child of this cell;- # any cell that defines a variable defined by this cell becomes- # a sibling.- for name, variable_data in cell.variable_data.items():- # NB. Only the last definition matters.- # Technically more nuanced with branching statements, but this is- # the best we can do with static analysis.- variable = variable_data[-1]- typed_def = (name, variable.kind)-- if (- name in self.definitions- and typed_def not in self.typed_definitions- ):- # Duplicate if the qualified name is no different- if (- variable.qualified_name == name- or variable.language != "sql"- ):- self.definitions[name].add(cell_id)- else:- self.definitions.setdefault(name, set()).add(cell_id)- self.typed_definitions.setdefault(typed_def, set()).add(- cell_id- )- self.definition_types.setdefault(name, set()).add(- variable.kind- )-- for sibling in self.definitions[name]:- # TODO(akshayka): Distinguish between Python/SQL?- if sibling != cell_id:- siblings.add(sibling)- self.siblings[sibling].add(cell_id)-- # a cell can refer to its own defs, but that doesn't add an- # edge to the dependency graph- referring_cells = self.get_referring_cells(- name,- language=variable_data[-1].language,- ) - set((cell_id,))- # we will add an edge (cell_id, v) for each v in- # referring_cells; if there is a path from v to cell_id, then- # the new edge will form a cycle- for v in referring_cells:- path = self.get_path(v, cell_id)- if path:- self.cycles.add(tuple([(cell_id, v)] + path))-- children.update(referring_cells)- for child in referring_cells:- self.parents[child].add(cell_id)-- # Next, we process the cells references. The cell becomes a child- # of cells that define its referenced variables. We also have- # special logic for handling references that are deleted by this cell,- # since cells that delete variables that were defined elsewhere- # are made children of cells that reference that variable.-- for name in cell.refs:- # First, for each referenced variable, we add cells that define- # that variable as parents- other_ids_defining_name: set[CellId_t] = (- self.definitions[name]- if name in self.definitions- else set()- ) - set((cell_id,))-- variable_name: Name = name-- # Handle SQL matching for hierarchical references- sql_matches: list[tuple[set[CellId_t], Name]] = []- sql_ref = cell.sql_refs.get(name)- if sql_ref:- sql_matches = self._find_sql_hierarchical_matches(sql_ref)- for matching_cell_ids, _ in sql_matches:- if cell_id in matching_cell_ids:- LOGGER.debug(- "Cell %s is referencing itself", cell_id- )- continue- other_ids_defining_name.update(matching_cell_ids)-- # If other_ids_defining_name is empty, the user will get a- # NameError at runtime (unless the symbol is a builtin).- for other_id in other_ids_defining_name:- if other_id == cell_id:- LOGGER.error("Cell %s is referencing itself", cell_id)- continue- if not self._is_valid_cell_reference(- other_id, variable_name- ):- continue- other_cell = self.cells[other_id]-- variable_name = self._resolve_variable_name(- variable_name, other_cell, sql_ref, sql_matches- )-- # If we don't have a matching variable name, skip- if variable_name not in other_cell.variable_data:- LOGGER.error(- "Variable %s is not defined in cell %s",- variable_name,- other_id,- )- continue-- other_variable_data = other_cell.variable_data[- variable_name- ][-1]- language = other_variable_data.language- if language == "sql" and cell.language == "python":- # SQL table/db def -> Python ref is not an edge- continue- if language == "sql" and cell.language == "sql":- # Edges between SQL cells need to respect hierarchy.- if sql_ref and not sql_ref.matches_hierarchical_ref(- variable_name,- other_variable_data.qualified_name or name,- kind=cast(SQLTypes, other_variable_data.kind),- ):- continue- parents.add(other_id)- # we are adding an edge (other_id, cell_id). If there- # is a path from cell_id to other_id, then the new- # edge forms a cycle- path = self.get_path(cell_id, other_id)- if path:- self.cycles.add(tuple([(other_id, cell_id)] + path))- self.children[other_id].add(cell_id)-- # Next, any cell that deletes this referenced variable is made- # a child of this cell. In particular, if a cell deletes a- # variable, it becomes a child of all other cells that- # reference that variable. This means that if two cells delete- # the same variable, they form a cycle.- #- # For example, two cells- #- # cell u: x- # cell v: del x- #- # v becomes a child of u.- #- # Another example:- #- # cell u: del x- # cell v: del x- #- # u and v form a cycle.- other_ids_deleting_name: set[CellId_t] = set(- cid- for cid in self.get_referring_cells(- name, language="python"- )- if name in self.cells[cid].deleted_refs- ) - set((cell_id,))- for v in other_ids_deleting_name:- path = self.get_path(v, cell_id)- if path:- self.cycles.add(tuple([(cell_id, v)] + path))- self.parents[v].add(cell_id)- children.update(other_ids_deleting_name)-- # Finally, if this cell deletes a variable, we make it a child of- # all other cells that reference this variable.- for name in cell.deleted_refs:- referring_cells = self.get_referring_cells(- name, language="python"- ) - set((cell_id,))- for other_id in referring_cells:- parents.add(other_id)- path = self.get_path(cell_id, other_id)- if path:- self.cycles.add(tuple([(other_id, cell_id)] + path))- self.children[other_id].add(cell_id)-- LOGGER.debug("Registered cell %s and released graph lock", cell_id)- if self.is_any_ancestor_stale(cell_id):- self.set_stale(set([cell_id]))-- if self.is_any_ancestor_disabled(cell_id):- cell.set_runtime_state(status="disabled-transitively")-- def is_any_ancestor_stale(self, cell_id: CellId_t) -> bool:- return any(self.cells[cid].stale for cid in self.ancestors(cell_id))-- def is_any_ancestor_disabled(self, cell_id: CellId_t) -> bool:- return any(- self.cells[cid].config.disabled for cid in self.ancestors(cell_id)- )-- def disable_cell(self, cell_id: CellId_t) -> None:- """- Disables a cell in the graph.-- Does not mutate the graph (but does mutate cell statuses).-- Returns the ids of descendants that are disabled transitively.- """- if cell_id not in self.cells:- raise ValueError(f"Cell {cell_id} not found")-- for cid in transitive_closure(self, set([cell_id])) - set([cell_id]):- cell = self.cells[cid]- cell.set_runtime_state(status="disabled-transitively")-- def enable_cell(self, cell_id: CellId_t) -> set[CellId_t]:- """- Enables a cell in the graph.-- Does not mutate the graph (but does mutate cell statuses).-- Returns:- - set of cells that were stale and should be re-run- """- if cell_id not in self.cells:- raise ValueError(f"Cell {cell_id} not found")-- cells_to_run: set[CellId_t] = set()- for cid in transitive_closure(self, set([cell_id])):- if not self.is_disabled(cid):- child = self.cells[cid]- if child.stale:- # cell was previously disabled, is no longer- # disabled, and is stale: needs to run.- cells_to_run.add(cid)- if child.disabled_transitively:- # cell is no longer disabled: status -> idle- child.set_runtime_state("idle")- return cells_to_run-- def delete_cell(self, cell_id: CellId_t) -> set[CellId_t]:- """Removes a cell from the graph.-- Mutates the graph, acquiring `self.lock`.-- Returns the ids of the children of the removed cell.- """- LOGGER.debug("Acquiring graph lock to delete cell %s", cell_id)- with self.lock:- LOGGER.debug("Acquired graph lock to delete cell %s", cell_id)- if cell_id not in self.cells:- raise ValueError(f"Cell {cell_id} not found")-- # Removing this cell from its defs' definer sets- for name in self.cells[cell_id].defs:- name_defs = self.definitions[name]- name_defs.remove(cell_id)- if not name_defs:- # No more cells define this name, so we remove it from the- # graph- del self.definitions[name]- # Clean up all typed definitions- for typed_def in self.definition_types[name]:- del self.typed_definitions[(name, typed_def)]- del self.definition_types[name]-- # Remove cycles that are broken from removing this cell.- edges = [(cell_id, child) for child in self.children[cell_id]] + [- (parent, cell_id) for parent in self.parents[cell_id]- ]- for e in edges:- broken_cycles = [c for c in self.cycles if e in c]- for c in broken_cycles:- self.cycles.remove(c)-- # Grab a reference to children before we remove it from our map.- children = self.children[cell_id]-- # Purge this cell from the graph.- del self.cells[cell_id]- del self.children[cell_id]- del self.parents[cell_id]- del self.siblings[cell_id]-- for elems in self.parents.values():- if cell_id in elems:- elems.remove(cell_id)- for elems in self.children.values():- if cell_id in elems:- elems.remove(cell_id)- for elems in self.siblings.values():- if cell_id in elems:- elems.remove(cell_id)- LOGGER.debug("Deleted cell %s and Released graph lock.", cell_id)- return children-- def is_disabled(self, cell_id: CellId_t) -> bool:- if cell_id not in self.cells:- raise ValueError(f"Cell {cell_id} not in graph.")- cell = self.cells[cell_id]- if cell.config.disabled:- return True- seen: set[CellId_t] = set()- queue = [cell_id]- while queue:- cid = queue.pop()- seen.add(cid)- for parent_id in self.parents[cid]:- if parent_id in seen:- continue- elif self.cells[parent_id].config.disabled:- return True- else:- queue.append(parent_id)- return False-- def get_imports(- self, cell_id: Optional[CellId_t] = None- ) -> dict[Name, ImportData]:- imports = {}- cells = (- self.cells.values() if cell_id is None else [self.cells[cell_id]]- )- for cell in cells:- for imported in cell.imports:- imports[imported.definition] = imported- return imports-- def get_multiply_defined(self) -> list[Name]:- """Return a list of names that are defined in multiple cells"""- names: list[Name] = []- for name, definers in self.definitions.items():- if len(definers) > 1:- names.append(name)- return names-- def get_deleted_nonlocal_ref(self) -> list[Name]:- names: list[Name] = []- for cell in self.cells.values():- for ref in cell.deleted_refs:- if ref in self.definitions:- names.append(ref)- return names-- def descendants(self, cell_id: CellId_t) -> set[CellId_t]:- return transitive_closure(self, set([cell_id]), inclusive=False)-- def ancestors(self, cell_id: CellId_t) -> set[CellId_t]:- return transitive_closure(- self, set([cell_id]), children=False, inclusive=False- )-- def set_stale(- self, cell_ids: set[CellId_t], prune_imports: bool = False- ) -> None:- relatives = None if not prune_imports else import_block_relatives- for cid in transitive_closure(self, cell_ids, relatives=relatives):- self.cells[cid].set_stale(stale=True)-- def get_stale(self) -> set[CellId_t]:- return set([cid for cid, cell in self.cells.items() if cell.stale])-- def get_transitive_references(- self,- refs: set[Name],- inclusive: bool = True,- predicate: Callable[[Name, VariableData], bool] | None = None,- ) -> set[Name]:- """Return a set of the passed-in cells' references and their- references on the block (function / class) level.-- If inclusive, includes the references of the passed-in cells in the- set.-- If predicate, only references satisfying predicate(ref) are included- """- # TODO: Consider caching on the graph level and updating on register /- # delete- processed: set[Name] = set()- queue: set[Name] = refs & self.definitions.keys()- predicate = predicate or (lambda *_: True)-- while queue:- # Should ideally be one cell per ref, but for completion, stay- # agnostic to potenital cycles.- cells = {- cell_id- for ref in queue- for cell_id in self.definitions.get(ref, set())- }-- for cell_id in cells:- data = self.cells[cell_id].variable_data- newly_processed = set(data.keys()) & queue- processed.update(newly_processed)- queue.difference_update(newly_processed)- for variable in newly_processed:- # variables can be defined multiple times in a single- # cell ...- for datum in data[variable]:- if predicate(variable, datum):- to_process = datum.required_refs - processed- queue.update(to_process & self.definitions.keys())- # Private variables referenced by public functions- # have to be included.- processed.update(- maybe_private- for maybe_private in (- to_process - self.definitions.keys()- )- if is_mangled_local(maybe_private, cell_id)- )-- if inclusive:- return processed | refs- return processed - refs-- def copy(self, filename: None | str = None) -> DirectedGraph:- """Return a deep copy of the graph by recompiling all cells.-- This is mainly useful in the case where recompilation must be done- due to a dynamically changing notebook, where the line cache must be- consistent with the cell code, e.g. for debugging.- """- from marimo._ast.compiler import compile_cell-- graph = DirectedGraph()- with self.lock:- for cid, old_cell in self.cells.items():- cell = compile_cell(- old_cell.code,- cell_id=cid,- filename=filename,- )- # Carry over import data manually- imported_defs = old_cell.import_workspace.imported_defs- is_import_block = old_cell.import_workspace.is_import_block- cell.import_workspace.imported_defs = imported_defs- cell.import_workspace.is_import_block = is_import_block- # Reregister- graph.register_cell(cid, cell)- return graph---def transitive_closure(- graph: DirectedGraph,- cell_ids: set[CellId_t],- children: bool = True,- inclusive: bool = True,- relatives: (- Callable[[DirectedGraph, CellId_t, bool], set[CellId_t]] | None- ) = None,- predicate: Callable[[CellImpl], bool] | None = None,-) -> set[CellId_t]:- """Return a set of the passed-in cells and their descendants or ancestors-- If children is True, returns descendants; otherwise, returns ancestors-- If inclusive, includes passed-in cells in the set.-- If relatives is not None, it computes the parents/children of a- cell-- If predicate, only cells satisfying predicate(cell) are included; applied- after the relatives are computed- """-- result: set[CellId_t] = cell_ids.copy() if inclusive else set()- seen: set[CellId_t] = cell_ids.copy()- queue: deque[CellId_t] = deque(cell_ids)- predicate = predicate or (lambda _: True)-- def _relatives(cid: CellId_t) -> set[CellId_t]:- if relatives is None:- return graph.children[cid] if children else graph.parents[cid]- return relatives(graph, cid, children)-- while queue:- cid = queue.popleft() # O(1) operation-- relatives_set = _relatives(cid)- new_relatives = relatives_set - seen-- if new_relatives:- # Add new relatives to queue and result if they pass predicate- for relative in new_relatives:- if predicate(graph.cells[relative]):- result.add(relative)- seen.add(relative)- queue.append(relative)-- return result---def induced_subgraph(- graph: DirectedGraph, cell_ids: Collection[CellId_t]-) -> tuple[dict[CellId_t, set[CellId_t]], dict[CellId_t, set[CellId_t]]]:- """Return parents and children for each node in `cell_ids`-- Represents the subgraph induced by `cell_ids`.- """- parents: dict[CellId_t, set[CellId_t]] = {}- children: dict[CellId_t, set[CellId_t]] = {}- for cid in cell_ids:- parents[cid] = set(p for p in graph.parents[cid] if p in cell_ids)- children[cid] = set(c for c in graph.children[cid] if c in cell_ids)- return parents, children---def get_cycles(- graph: DirectedGraph, cell_ids: Collection[CellId_t]-) -> list[tuple[Edge, ...]]:- """Get all cycles among `cell_ids`."""- _, induced_children = induced_subgraph(graph, cell_ids)- induced_edges = set(- [(u, v) for u in induced_children for v in induced_children[u]]- )- return [c for c in graph.cycles if all(e in induced_edges for e in c)]---def topological_sort(- graph: DirectedGraph, cell_ids: Collection[CellId_t]-) -> list[CellId_t]:- """Sort `cell_ids` in a topological order using a heap queue.-- When multiple cells have the same parents (including no parents), the tie is broken by- registration order - cells registered earlier are processed first.- """- from heapq import heapify, heappop, heappush-- # Use a list for O(1) lookup of registration order- registration_order = list(graph.cells.keys())- top_down_keys = {key: idx for idx, key in enumerate(registration_order)}-- # Build adjacency lists and in-degree counts- parents, children = induced_subgraph(graph, cell_ids)- in_degree = {cid: len(parents[cid]) for cid in cell_ids}-- # Initialize heap with roots- heap = [- (top_down_keys[cid], cid) for cid in cell_ids if in_degree[cid] == 0- ]- heapify(heap)-- sorted_cell_ids: list[CellId_t] = []- while heap:- _, cid = heappop(heap)- sorted_cell_ids.append(cid)-- # Process children- for child in children[cid]:- in_degree[child] -= 1- if in_degree[child] == 0:- heappush(heap, (top_down_keys[child], child))-- return sorted_cell_ids---def prune_cells_for_overrides(- graph: DirectedGraph,- execution_order: Collection[CellId_t],- overrides: dict[str, Any],-) -> list[CellId_t]:- """Prune cells from execution when their definitions are overridden.-- When variable definitions are provided externally (overrides), this function- identifies cells that would normally define those variables and removes them- from the execution order. It also validates that all definitions from pruned- cells are provided in the overrides.-- Args:- graph: The dataflow graph containing cell dependencies- execution_order: Ordered collection of cells to execute- overrides: Dictionary mapping variable names to their override values-- Returns:- Filtered execution order excluding cells whose definitions are overridden-- Raises:- IncompleteRefsError: If overrides don't provide all definitions from- pruned cells (e.g., a cell defines both x and y, but only x is- provided in overrides)-- Example:- If cell A defines variables x and y, and overrides = {"x": 1, "y": 2},- then cell A will be pruned from execution_order. However, if overrides- only contains {"x": 1}, an IncompleteRefsError is raised because y is- missing.- """- if not overrides:- return list(execution_order)-- cells_to_prune: set[CellId_t] = set()- refs = set(overrides.keys())-- # Find cells that define the overridden variables- for ref_name in refs:- if ref_name in graph.definitions:- defining_cells = graph.get_defining_cells(ref_name)- cells_to_prune.update(defining_cells)-- # Validate that all definitions from pruned cells are provided- missing_defs: set[str] = set()- for cell_id in cells_to_prune:- cell = graph.cells[cell_id]- # Check all definitions this cell would have provided- for missing in cell.defs - refs:- missing_defs.add(missing)-- if missing_defs:- from marimo._ast.errors import IncompleteRefsError-- raise IncompleteRefsError(- f"When providing refs that override cell definitions, you must "- f"provide all definitions from those cells. Missing: {sorted(missing_defs)}. "- f"Provided refs: {sorted(refs)}."- )-- # Return filtered execution order- return [cid for cid in execution_order if cid not in cells_to_prune]---def import_block_relatives(- graph: DirectedGraph, cid: CellId_t, children: bool-) -> set[CellId_t]:- if not children:- return graph.parents[cid]-- cell = graph.cells[cid]- if not cell.import_workspace.is_import_block:- return graph.children[cid]-- # This cell is an import block, which should be special cased:- #- # We prune definitions that have already been imported from the set of- # definitions used to find the descendants of this cell.- unimported_defs = cell.defs - cell.import_workspace.imported_defs-- children_ids = {- child_id- for name in unimported_defs- for child_id in graph.get_referring_cells(name, language="python")- }-- # If children haven't been executed, then still use imported defs;- # handle an edge case when an import cell is interrupted by an- # exception or user interrupt, so that a module is imported but the- # cell's children haven't run.- if cell.import_workspace.imported_defs:- interrupted_states = {"interrupted", "cancelled", "marimo-error", None}- children_ids.update(- child_id- for name in cell.import_workspace.imported_defs- for child_id in graph.get_referring_cells(name, language="python")- if graph.cells[child_id].run_result_status in interrupted_states- )-- return children_ids---class Runner:- """Utility for running individual cells in a graph-- This class provides methods to a run a cell in the graph and obtain its- output (last expression) and the values of its defs.-- If needed, the runner will recursively compute the values of the cell's- refs by executing its ancestors. Refs can also be substituted by the- caller.-- TODO(akshayka): Add an API for caching defs across cell runs.- """-- def __init__(self, graph: DirectedGraph) -> None:- self._graph = graph- self._executor = get_executor(ExecutionConfig())-- @staticmethod- def _returns(cell_impl: CellImpl, glbls: dict[str, Any]) -> dict[str, Any]:- return {name: glbls[name] for name in cell_impl.defs if name in glbls}-- @staticmethod- def _substitute_refs(- cell_impl: CellImpl,- glbls: dict[str, Any],- kwargs: dict[str, Any],- ) -> None:- for argname, argvalue in kwargs.items():- if argname in cell_impl.refs:- glbls[argname] = argvalue- else:- raise ValueError(- f"Cell got unexpected argument {argname}"- f"The allowed arguments are {cell_impl.refs}."- )-- def _get_ancestors(- self, cell_impl: CellImpl, kwargs: dict[str, Any]- ) -> set[CellId_t]:- # Get the transitive closure of parents defining unsubstituted refs- graph = self._graph- substitutions = set(kwargs.keys())- unsubstituted_refs = cell_impl.refs - substitutions- parent_ids = set(- [- parent_id- for parent_id in graph.parents[cell_impl.cell_id]- if graph.cells[parent_id].defs.intersection(unsubstituted_refs)- ]- )- return transitive_closure(graph, parent_ids, children=False)-- @staticmethod- def _validate_kwargs(cell_impl: CellImpl, kwargs: dict[str, Any]) -> None:- for argname in kwargs:- if argname not in cell_impl.refs:- raise ValueError(- f"Cell got unexpected argument {argname}; "- f"The allowed arguments are {cell_impl.refs}."- )-- def is_coroutine(self, cell_id: CellId_t) -> bool:- return self._graph.cells[cell_id].is_coroutine() or any(- self._graph.cells[cid].is_coroutine()- for cid in self._get_ancestors(- self._graph.cells[cell_id], kwargs={}- )- )-- async def run_cell_async(- self, cell_id: CellId_t, kwargs: dict[str, Any]- ) -> tuple[Any, dict[str, Any]]:- """Run a possibly async cell and its ancestors-- Substitutes kwargs as refs for the cell, omitting ancestors that- whose refs are substituted.- """- graph = self._graph- cell_impl = graph.cells[cell_id]- Runner._validate_kwargs(cell_impl, kwargs)- ancestor_ids = self._get_ancestors(cell_impl, kwargs)-- glbls: dict[str, Any] = {}- for cid in topological_sort(graph, ancestor_ids):- await self._executor.execute_cell_async(- graph.cells[cid], glbls, graph- )-- Runner._substitute_refs(cell_impl, glbls, kwargs)- output = await self._executor.execute_cell_async(- graph.cells[cell_impl.cell_id], glbls, graph- )- defs = Runner._returns(cell_impl, glbls)- return output, defs-- def run_cell_sync(- self, cell_id: CellId_t, kwargs: dict[str, Any]- ) -> tuple[Any, dict[str, Any]]:- """Run a synchronous cell and its ancestors-- Substitutes kwargs as refs for the cell, omitting ancestors that- whose refs are substituted.-- Raises a `RuntimeError` if the cell or any of its unsubstituted- ancestors are coroutine functions.- """- graph = self._graph- cell_impl = graph.cells[cell_id]- if cell_impl.is_coroutine():- raise RuntimeError(- "A coroutine function can't be run synchronously. "- "Use `run_async()` instead"- )-- Runner._validate_kwargs(cell_impl, kwargs)- ancestor_ids = self._get_ancestors(cell_impl, kwargs)-- if any(graph.cells[cid].is_coroutine() for cid in ancestor_ids):- raise RuntimeError(- "Cell has an ancestor that is a "- "coroutine (async) cell. Use `run_async()` instead"- )-- glbls: dict[str, Any] = {}- for cid in topological_sort(graph, ancestor_ids):- self._executor.execute_cell(graph.cells[cid], glbls, graph)-- self._substitute_refs(cell_impl, glbls, kwargs)- output = self._executor.execute_cell(- graph.cells[cell_impl.cell_id], glbls, graph- )- defs = Runner._returns(cell_impl, glbls)- return output, defsdiff --git a/marimo/_runtime/dataflow/__init__.py b/marimo/_runtime/dataflow/__init__.pynew file mode 100644index 0000000000..94c4abd5c5--- /dev/null+++ b/marimo/_runtime/dataflow/__init__.py@@ -0,0 +1,259 @@+# Copyright 2024 Marimo. All rights reserved.+from __future__ import annotations++from collections import deque+from typing import TYPE_CHECKING, Any, Callable++from marimo import _loggers+from marimo._ast.cell import CellImpl+from marimo._runtime.dataflow.graph import DirectedGraph+from marimo._runtime.dataflow.runner import Runner+from marimo._runtime.dataflow.topology import GraphTopology+from marimo._runtime.dataflow.types import Edge, EdgeWithVar+from marimo._types.ids import CellId_t++if TYPE_CHECKING:+ from collections.abc import Collection+++LOGGER = _loggers.marimo_logger()+++def transitive_closure(+ graph: GraphTopology,+ cell_ids: set[CellId_t],+ *,+ children: bool = True,+ inclusive: bool = True,+ relatives: Callable[[CellId_t, bool], set[CellId_t]] | None = None,+ predicate: Callable[[CellImpl], bool] | None = None,+) -> set[CellId_t]:+ """Return a set of the passed-in cells and their descendants or ancestors++ If children is True, returns descendants; otherwise, returns ancestors++ If inclusive, includes passed-in cells in the set.++ If relatives is not None, it computes the parents/children of a+ cell++ If predicate, only cells satisfying predicate(cell) are included; applied+ after the relatives are computed+ """++ result: set[CellId_t] = cell_ids.copy() if inclusive else set()+ seen: set[CellId_t] = cell_ids.copy()+ queue: deque[CellId_t] = deque(cell_ids)+ predicate = predicate or (lambda _: True)++ def _relatives(cid: CellId_t) -> set[CellId_t]:+ if relatives is None:+ return graph.children[cid] if children else graph.parents[cid]+ return relatives(cid, children)++ while queue:+ cid = queue.popleft() # O(1) operation++ relatives_set = _relatives(cid)+ new_relatives = relatives_set - seen++ if new_relatives:+ # Add new relatives to queue and result if they pass predicate+ for relative in new_relatives:+ if predicate(graph.cells[relative]):+ result.add(relative)+ seen.add(relative)+ queue.append(relative)++ return result+++def induced_subgraph(+ graph: GraphTopology, cell_ids: Collection[CellId_t]+) -> tuple[dict[CellId_t, set[CellId_t]], dict[CellId_t, set[CellId_t]]]:+ """Return parents and children for each node in `cell_ids`++ Represents the subgraph induced by `cell_ids`.+ """+ parents: dict[CellId_t, set[CellId_t]] = {}+ children: dict[CellId_t, set[CellId_t]] = {}+ for cid in cell_ids:+ parents[cid] = set(p for p in graph.parents[cid] if p in cell_ids)+ children[cid] = set(c for c in graph.children[cid] if c in cell_ids)+ return parents, children+++def get_cycles(+ graph: DirectedGraph, cell_ids: Collection[CellId_t]+) -> list[tuple[Edge, ...]]:+ """Get all cycles among `cell_ids`."""+ _, induced_children = induced_subgraph(graph, cell_ids)+ induced_edges = set(+ [(u, v) for u in induced_children for v in induced_children[u]]+ )+ return [c for c in graph.cycles if all(e in induced_edges for e in c)]+++def topological_sort(+ graph: GraphTopology, cell_ids: Collection[CellId_t]+) -> list[CellId_t]:+ """Sort `cell_ids` in a topological order using a heap queue.++ When multiple cells have the same parents (including no parents), the tie is broken by+ registration order - cells registered earlier are processed first.+ """+ from heapq import heapify, heappop, heappush++ # Use a list for O(1) lookup of registration order+ registration_order = list(graph.cells.keys())+ top_down_keys = {key: idx for idx, key in enumerate(registration_order)}++ # Build adjacency lists and in-degree counts+ parents, children = induced_subgraph(graph, cell_ids)+ in_degree = {cid: len(parents[cid]) for cid in cell_ids}++ # Initialize heap with roots+ heap = [+ (top_down_keys[cid], cid) for cid in cell_ids if in_degree[cid] == 0+ ]+ heapify(heap)++ sorted_cell_ids: list[CellId_t] = []+ while heap:+ _, cid = heappop(heap)+ sorted_cell_ids.append(cid)++ # Process children+ for child in children[cid]:+ in_degree[child] -= 1+ if in_degree[child] == 0:+ heappush(heap, (top_down_keys[child], child))++ return sorted_cell_ids+++def prune_cells_for_overrides(+ graph: DirectedGraph,+ execution_order: Collection[CellId_t],+ overrides: dict[str, Any],+) -> list[CellId_t]:+ """Prune cells from execution when their definitions are overridden.++ When variable definitions are provided externally (overrides), this function+ identifies cells that would normally define those variables and removes them+ from the execution order. It also validates that all definitions from pruned+ cells are provided in the overrides.++ Args:+ graph: The dataflow graph containing cell dependencies+ execution_order: Ordered collection of cells to execute+ overrides: Dictionary mapping variable names to their override values++ Returns:+ Filtered execution order excluding cells whose definitions are overridden++ Raises:+ IncompleteRefsError: If overrides don't provide all definitions from+ pruned cells (e.g., a cell defines both x and y, but only x is+ provided in overrides)++ Example:+ If cell A defines variables x and y, and overrides = {"x": 1, "y": 2},+ then cell A will be pruned from execution_order. However, if overrides+ only contains {"x": 1}, an IncompleteRefsError is raised because y is+ missing.+ """+ if not overrides:+ return list(execution_order)++ cells_to_prune: set[CellId_t] = set()+ refs = set(overrides.keys())++ # Find cells that define the overridden variables+ for ref_name in refs:+ if ref_name in graph.definitions:+ defining_cells = graph.get_defining_cells(ref_name)+ cells_to_prune.update(defining_cells)++ # Validate that all definitions from pruned cells are provided+ missing_defs: set[str] = set()+ for cell_id in cells_to_prune:+ cell = graph.cells[cell_id]+ # Check all definitions this cell would have provided+ for missing in cell.defs - refs:+ missing_defs.add(missing)++ if missing_defs:+ from marimo._ast.errors import IncompleteRefsError++ raise IncompleteRefsError(+ f"When providing refs that override cell definitions, you must "+ f"provide all definitions from those cells. Missing: {sorted(missing_defs)}. "+ f"Provided refs: {sorted(refs)}."+ )++ # Return filtered execution order+ return [cid for cid in execution_order if cid not in cells_to_prune]+++def get_import_block_relatives(+ graph: DirectedGraph,+) -> Callable[[CellId_t, bool], set[CellId_t]]:+ def import_block_relatives(cid: CellId_t, children: bool) -> set[CellId_t]:+ if not children:+ return graph.parents[cid]++ cell = graph.cells[cid]+ if not cell.import_workspace.is_import_block:+ return graph.children[cid]++ # This cell is an import block, which should be special cased:+ #+ # We prune definitions that have already been imported from the set of+ # definitions used to find the descendants of this cell.+ unimported_defs = cell.defs - cell.import_workspace.imported_defs++ children_ids = {+ child_id+ for name in unimported_defs+ for child_id in graph.get_referring_cells(name, language="python")+ }++ # If children haven't been executed, then still use imported defs;+ # handle an edge case when an import cell is interrupted by an+ # exception or user interrupt, so that a module is imported but the+ # cell's children haven't run.+ if cell.import_workspace.imported_defs:+ interrupted_states = {+ "interrupted",+ "cancelled",+ "marimo-error",+ None,+ }+ children_ids.update(+ child_id+ for name in cell.import_workspace.imported_defs+ for child_id in graph.get_referring_cells(+ name, language="python"+ )+ if graph.cells[child_id].run_result_status+ in interrupted_states+ )++ return children_ids++ return import_block_relatives+++__all__ = [+ "Edge",+ "EdgeWithVar",+ "DirectedGraph",+ "Runner",+ "get_cycles",+ "get_import_block_relatives",+ "induced_subgraph",+ "prune_cells_for_overrides",+ "topological_sort",+ "transitive_closure",+]diff --git a/marimo/_runtime/dataflow/cycles.py b/marimo/_runtime/dataflow/cycles.pynew file mode 100644index 0000000000..41ee9f2358--- /dev/null+++ b/marimo/_runtime/dataflow/cycles.py@@ -0,0 +1,91 @@+# Copyright 2024 Marimo. All rights reserved.+"""Cycle detection and tracking for cell dependencies."""++from __future__ import annotations++from dataclasses import dataclass, field+from typing import TYPE_CHECKING++from marimo._runtime.dataflow.types import Edge++if TYPE_CHECKING:+ from collections.abc import Collection++ from marimo._runtime.dataflow.topology import GraphTopology+ from marimo._types.ids import CellId_t+++@dataclass+class CycleTracker:+ """Detects and tracks cycles in the graph.++ Responsibilities:+ - Detect cycles when edges are added+ - Maintain cycle set+ - Remove broken cycles+ """++ # The set of cycles in the graph+ # Each cycle is represented as a tuple of edges+ cycles: set[tuple[Edge, ...]] = field(default_factory=set)++ def detect_cycle_for_edge(+ self,+ edge: Edge,+ topology: GraphTopology,+ ) -> tuple[Edge, ...] | None:+ """Detect if adding an edge creates a cycle.++ Args:+ edge: The edge (parent, child) being added+ topology: The graph topology to search for paths++ Returns:+ The cycle as a tuple of edges if one exists, None otherwise.+ The cycle includes the new edge plus the path from child back to parent.+ """+ parent, child = edge+ # Check if there's a path from child back to parent+ # If so, adding this edge creates a cycle+ path = topology.get_path(child, parent)+ if path:+ # The cycle is: edge + path+ cycle = tuple([edge] + path)+ self.cycles.add(cycle)+ return cycle+ return None++ def remove_cycles_with_edge(self, edge: Edge) -> None:+ """Remove all cycles that contain the given edge.++ This should be called when an edge is removed from the graph.+ """+ broken_cycles = [c for c in self.cycles if edge in c]+ for c in broken_cycles:+ self.cycles.remove(c)++ def get_cycles(+ self,+ cell_ids: Collection[CellId_t],+ topology: GraphTopology,+ ) -> list[tuple[Edge, ...]]:+ """Get all cycles among the given cell_ids.++ Args:+ cell_ids: The cells to consider+ topology: The graph topology++ Returns:+ List of cycles, where each cycle is a tuple of edges.+ Only returns cycles where all edges are between cells in cell_ids.+ """+ # Build induced subgraph edges+ induced_edges = set()+ for u in cell_ids:+ if u in topology.children:+ for v in topology.children[u]:+ if v in cell_ids:+ induced_edges.add((u, v))++ # Filter cycles to those in the induced subgraph+ return [c for c in self.cycles if all(e in induced_edges for e in c)]diff --git a/marimo/_runtime/dataflow/definitions.py b/marimo/_runtime/dataflow/definitions.pynew file mode 100644index 0000000000..8f1e71576c--- /dev/null+++ b/marimo/_runtime/dataflow/definitions.py@@ -0,0 +1,153 @@+# Copyright 2024 Marimo. All rights reserved.+"""Variable definition tracking for cells."""++from __future__ import annotations++from dataclasses import dataclass, field+from typing import TYPE_CHECKING++if TYPE_CHECKING:+ from marimo._ast.sql_visitor import SQLRef+ from marimo._ast.visitor import Name, VariableData+ from marimo._types.ids import CellId_t+++@dataclass+class DefinitionRegistry:+ """Tracks variable definitions across cells.++ Responsibilities:+ - Track which cells define which variables+ - Handle typed definitions (SQL vs Python)+ - Detect multiply-defined names+ - SQL hierarchical reference matching+ """++ # A mapping from defs to the cells that define them+ definitions: dict[Name, set[CellId_t]] = field(default_factory=dict)++ # Typed definitions for SQL support: (name, type) -> cell_ids+ # e.g. ("my_table", "table") -> {cell_id_1}+ typed_definitions: dict[tuple[Name, str], set[CellId_t]] = field(+ default_factory=dict+ )++ # Track all types for a given definition name+ # e.g. "my_table" -> {"table", "view"}+ definition_types: dict[Name, set[str]] = field(default_factory=dict)++ def register_definition(+ self,+ cell_id: CellId_t,+ name: Name,+ variable_data: list[VariableData],+ ) -> set[CellId_t]:+ """Register a definition for a cell.++ Args:+ cell_id: The cell that defines the variable+ name: The variable name+ variable_data: List of metadata about the variable (from cell.variable_data[name])++ Returns:+ Set of sibling cell IDs (cells that also define this variable)+ """+ variable = variable_data[-1] # Only the last definition matters+ typed_def = (name, variable.kind)++ # Check if this is a duplicate definition+ if (+ name in self.definitions+ and typed_def not in self.typed_definitions+ ):+ # Duplicate if the qualified name is no different+ if variable.qualified_name == name or variable.language != "sql":+ self.definitions[name].add(cell_id)+ else:+ self.definitions.setdefault(name, set()).add(cell_id)++ self.typed_definitions.setdefault(typed_def, set()).add(cell_id)+ self.definition_types.setdefault(name, set()).add(variable.kind)++ # Return siblings (other cells that define this name)+ siblings = self.definitions[name] - {cell_id}+ return siblings++ def unregister_definitions(+ self,+ cell_id: CellId_t,+ defs: set[Name],+ ) -> None:+ """Unregister all definitions for a cell.++ Args:+ cell_id: The cell being removed+ defs: The set of variable names defined by the cell+ """+ for name in defs:+ if name not in self.definitions:+ continue++ name_defs = self.definitions[name]+ name_defs.discard(cell_id)++ if not name_defs:+ # No more cells define this name, so we remove it+ del self.definitions[name]+ # Clean up all typed definitions+ for typed_def in self.definition_types.get(name, set()):+ self.typed_definitions.pop((name, typed_def), None)+ self.definition_types.pop(name, None)++ def get_defining_cells(self, name: Name) -> set[CellId_t]:+ """Get all cells that define a variable name.++ This is a singleton for well-formed graphs (no multiply-defined names).++ Args:+ name: The variable name++ Returns:+ Set of cell IDs that define this name+ """+ return self.definitions.get(name, set())++ def find_sql_hierarchical_matches(+ self, sql_ref: SQLRef+ ) -> list[tuple[set[CellId_t], Name]]:+ """Find cells that define components of a hierarchical SQL reference.++ This method searches through all definitions in the graph to find cells+ that define the individual components (table, schema, or catalog) of the+ hierarchical reference.++ For example, given a reference "my_schema.my_table", this method will:+ - Look for cells that define a table/view named "my_table"+ - Look for cells that define a catalog named "my_schema"+ (when the reference has at least 2 parts)++ Args:+ sql_ref: A hierarchical SQL reference (e.g., "schema.table",+ "catalog.schema.table") to find matching definitions for.++ Returns:+ A list of tuples containing:+ - A set of cell IDs that define components of the hierarchical reference+ - The definition of the name that was found (e.g., "schema.table" -> "table")+ """+ matching_cell_ids_list = []++ for (def_name, kind), cell_ids in self.typed_definitions.items():+ # Match table/view definitions+ if sql_ref.contains_hierarchical_ref(def_name, kind):+ matching_cell_ids_list.append((cell_ids, def_name))++ return matching_cell_ids_list++ def get_multiply_defined(self) -> list[Name]:+ """Return a list of names that are defined in multiple cells."""+ names: list[Name] = []+ for name, definers in self.definitions.items():+ if len(definers) > 1:+ names.append(name)+ return namesdiff --git a/marimo/_runtime/dataflow/edges.py b/marimo/_runtime/dataflow/edges.pynew file mode 100644index 0000000000..34f0d6308b--- /dev/null+++ b/marimo/_runtime/dataflow/edges.py@@ -0,0 +1,269 @@+# Copyright 2024 Marimo. All rights reserved.+"""Edge computation logic for cell dependencies."""++from __future__ import annotations++from typing import TYPE_CHECKING, Literal, Optional, cast++from marimo import _loggers++if TYPE_CHECKING:+ from marimo._ast.cell import CellImpl+ from marimo._ast.sql_visitor import SQLRef+ from marimo._ast.visitor import Name+ from marimo._runtime.dataflow.definitions import DefinitionRegistry+ from marimo._runtime.dataflow.topology import GraphTopology+ from marimo._types.ids import CellId_t++from marimo._ast.sql_visitor import SQLTypes++LOGGER = _loggers.marimo_logger()+++class EdgeComputer:+ """Computes graph edges based on cell definitions and references.++ Responsibilities:+ - Compute parent/child relationships from defs/refs+ - Handle SQL reference resolution+ - Handle deleted reference semantics+ - Apply language-specific edge rules (Python/SQL)+ """++ def __init__(+ self,+ topology: GraphTopology,+ definitions: DefinitionRegistry,+ ):+ self.topology = topology+ self.definitions = definitions++ def get_referring_cells(+ self, name: Name, language: Literal["python", "sql"]+ ) -> set[CellId_t]:+ """Get all cells that have a ref to `name`.++ The variable can be either a Python variable or a SQL variable (table).+ SQL variables don't leak to Python cells, but Python variables do leak+ to SQL.++ Only does a local analysis of refs, without taking into consideration+ whether refs are defined by other cells.+ """+ if language == "sql":+ # For SQL, only return SQL cells that reference the name+ cells = set()+ for cid, cell in self.topology.cells.items():+ if cell.language != "sql":+ continue++ for ref in cell.refs:+ # Direct reference match+ if ref == name:+ cells.add(cid)+ break++ sql_ref = cell.sql_refs.get(ref)++ kind: SQLTypes = "any"+ if name in cell.variable_data:+ variable_data = cell.variable_data[name][-1]+ kind = cast(SQLTypes, variable_data.kind)++ # Hierarchical reference match+ if sql_ref and sql_ref.matches_hierarchical_ref(+ name, ref, kind+ ):+ cells.add(cid)+ break++ return cells+ else:+ # For Python, return all cells that reference the name+ return {+ cid+ for cid, cell in self.topology.cells.items()+ if name in cell.refs+ }++ def _is_valid_cell_reference(+ self, cell_id: CellId_t, variable_name: Name+ ) -> bool:+ """Check if a cell reference is valid and log errors if not."""+ if cell_id not in self.topology.cells:+ LOGGER.error(+ "Variable %s is defined in cell %s, but is not in the graph",+ variable_name,+ cell_id,+ )+ return False+ return True++ def _resolve_variable_name(+ self,+ name: Name,+ other_cell: CellImpl,+ sql_ref: Optional[SQLRef],+ sql_matches: list[tuple[set[CellId_t], Name]],+ ) -> Name:+ """Resolve the variable name to use when checking if it exists in another cell.++ For regular (non-SQL) references, returns the original name unchanged.+ For SQL hierarchical references, finds the variable name from sql_matches that+ is actually defined in the other_cell.++ Example:+ cell_1: CREATE SCHEMA schema_name+ cell_2: CREATE TABLE schema_name.table_name+ cell_3: FROM schema_name.table_name SELECT *++ When cell_3 references "schema_name.table_name":+ - For cell_1: returns "schema_name" (what cell_1 actually defines)+ - For cell_2: returns "table_name" (what cell_2 actually defines)+ """+ if not sql_ref or name in other_cell.variable_data:+ return name++ # For SQL hierarchical references, find the matching variable name+ for _, matching_variable_name in sql_matches:+ if matching_variable_name in other_cell.variable_data:+ return matching_variable_name++ return name++ def compute_edges_for_cell(+ self, cell_id: CellId_t, cell: CellImpl+ ) -> tuple[set[CellId_t], set[CellId_t]]:+ """Compute parent and child edges for a cell being registered.++ Args:+ cell_id: The ID this cell is being registered with+ cell: The cell to compute edges for++ Returns:+ Tuple of (parents, children) where:+ - parents: cells that this cell depends on+ - children: cells that depend on this cell+ """+ parents: set[CellId_t] = set()+ children: set[CellId_t] = set()++ # First, process the variables that this cell defines. Any cell+ # that refers to a defined variable becomes a child of this cell.+ for name, variable_data in cell.variable_data.items():+ # NB. Only the last definition matters.+ # Technically more nuanced with branching statements, but this is+ # the best we can do with static analysis.+ variable = variable_data[-1]++ # a cell can refer to its own defs, but that doesn't add an+ # edge to the dependency graph+ referring_cells = self.get_referring_cells(+ name,+ language=variable.language,+ ) - {cell_id}++ children.update(referring_cells)++ # Next, process the cells references. The cell becomes a child+ # of cells that define its referenced variables. We also have+ # special logic for handling references that are deleted by this cell,+ # since cells that delete variables that were defined elsewhere+ # are made children of cells that reference that variable.+ for name in cell.refs:+ # First, for each referenced variable, we add cells that define+ # that variable as parents+ other_ids_defining_name: set[CellId_t] = (+ self.definitions.definitions.get(name, set()) - {cell_id}+ )++ variable_name: Name = name++ # Handle SQL matching for hierarchical references+ sql_matches: list[tuple[set[CellId_t], Name]] = []+ sql_ref = cell.sql_refs.get(name)+ if sql_ref:+ sql_matches = self.definitions.find_sql_hierarchical_matches(+ sql_ref+ )+ for matching_cell_ids, _ in sql_matches:+ if cell_id in matching_cell_ids:+ LOGGER.debug("Cell %s is referencing itself", cell_id)+ continue+ other_ids_defining_name.update(matching_cell_ids)++ # If other_ids_defining_name is empty, the user will get a+ # NameError at runtime (unless the symbol is a builtin).+ for other_id in other_ids_defining_name:+ if other_id == cell_id:+ LOGGER.error("Cell %s is referencing itself", cell_id)+ continue+ if not self._is_valid_cell_reference(other_id, variable_name):+ continue+ other_cell = self.topology.cells[other_id]++ variable_name = self._resolve_variable_name(+ variable_name, other_cell, sql_ref, sql_matches+ )++ # If we don't have a matching variable name, skip+ if variable_name not in other_cell.variable_data:+ LOGGER.error(+ "Variable %s is not defined in cell %s",+ variable_name,+ other_id,+ )+ continue++ other_variable_data = other_cell.variable_data[variable_name][+ -1+ ]+ language = other_variable_data.language+ if language == "sql" and cell.language == "python":+ # SQL table/db def -> Python ref is not an edge+ continue+ if language == "sql" and cell.language == "sql":+ # Edges between SQL cells need to respect hierarchy.+ if sql_ref and not sql_ref.matches_hierarchical_ref(+ variable_name,+ other_variable_data.qualified_name or name,+ kind=cast(SQLTypes, other_variable_data.kind),+ ):+ continue+ parents.add(other_id)++ # Next, any cell that deletes this referenced variable is made+ # a child of this cell. In particular, if a cell deletes a+ # variable, it becomes a child of all other cells that+ # reference that variable. This means that if two cells delete+ # the same variable, they form a cycle.+ #+ # For example, two cells+ #+ # cell u: x+ # cell v: del x+ #+ # v becomes a child of u.+ #+ # Another example:+ #+ # cell u: del x+ # cell v: del x+ #+ # u and v form a cycle.+ other_ids_deleting_name: set[CellId_t] = {+ cid+ for cid in self.get_referring_cells(name, language="python")+ if name in self.topology.cells[cid].deleted_refs+ } - {cell_id}+ children.update(other_ids_deleting_name)++ # Finally, if this cell deletes a variable, we make it a child of+ # all other cells that reference this variable.+ for name in cell.deleted_refs:+ referring_cells = self.get_referring_cells(+ name, language="python"+ ) - {cell_id}+ parents.update(referring_cells)++ return parents, childrendiff --git a/marimo/_runtime/dataflow/graph.py b/marimo/_runtime/dataflow/graph.pynew file mode 100644index 0000000000..09aa71f5f2--- /dev/null+++ b/marimo/_runtime/dataflow/graph.py@@ -0,0 +1,442 @@+# Copyright 2024 Marimo. All rights reserved.+"""Graph coordinator that orchestrates all dataflow components."""++from __future__ import annotations++import threading+from dataclasses import dataclass, field+from typing import TYPE_CHECKING, Callable, Literal, Optional++from marimo import _loggers+from marimo._ast.compiler import code_key+from marimo._ast.variables import is_mangled_local+from marimo._runtime.dataflow.cycles import CycleTracker+from marimo._runtime.dataflow.definitions import DefinitionRegistry+from marimo._runtime.dataflow.edges import EdgeComputer+from marimo._runtime.dataflow.topology import (+ GraphTopology,+ MutableGraphTopology,+)++if TYPE_CHECKING:+ from collections.abc import Mapping++ from marimo._ast.cell import CellImpl+ from marimo._ast.visitor import ImportData, Name, VariableData+ from marimo._types.ids import CellId_t++LOGGER = _loggers.marimo_logger()+++@dataclass(frozen=True)+class DirectedGraph(GraphTopology):+ """Main entry point that coordinates all graph operations.++ Responsibilities:+ - Coordinate topology, definitions, cycles, siblings+ - Execute register_cell/delete_cell operations+ - Maintain thread safety+ - Delegate to specialists+ """++ topology: MutableGraphTopology = field(+ default_factory=MutableGraphTopology+ )+ definition_registry: DefinitionRegistry = field(+ default_factory=DefinitionRegistry+ )+ cycle_tracker: CycleTracker = field(default_factory=CycleTracker)++ # Cells that define the same name+ #+ # siblings[cell_id] is a set of cell ids, one for each cell that shares a+ # definition with cell_id.+ #+ # If this dict is non-empty, then the marimo program contains multiply+ # defined names (and is therefore in an error state)+ # TODO: remove this field (might be unused)+ siblings: dict[CellId_t, set[CellId_t]] = field(default_factory=dict)++ # This lock must be acquired during methods that mutate the graph; it's+ # only needed because a graph is shared between the kernel and the code+ # completion service. It should almost always be uncontended.+ lock: threading.Lock = field(default_factory=threading.Lock)++ def is_cell_cached(self, cell_id: CellId_t, code: str) -> bool:+ """Whether a cell with id `cell_id` and code `code` is in the graph."""+ return (+ cell_id in self.topology.cells+ and code_key(code) == self.topology.cells[cell_id].key+ )++ def get_defining_cells(self, name: Name) -> set[CellId_t]:+ """Get all cells that define name.++ This is a singleton for well-formed graphs.+ """+ return self.definition_registry.get_defining_cells(name)++ def get_referring_cells(+ self, name: Name, language: Literal["python", "sql"]+ ) -> set[CellId_t]:+ """Get all cells that have a ref to `name`.++ The variable can be either a Python variable or a SQL variable (table).+ SQL variables don't leak to Python cells, but Python variables do leak+ to SQL.++ Only does a local analysis of refs, without taking into consideration+ whether refs are defined by other cells.+ """+ edge_computer = EdgeComputer(self.topology, self.definition_registry)+ return edge_computer.get_referring_cells(name, language)++ def register_cell(self, cell_id: CellId_t, cell: CellImpl) -> None:+ """Add a cell to the graph.++ Mutates the graph, acquiring `self.lock`.++ Requires that `cell_id` is not already in the graph.+ """+ LOGGER.debug("Acquiring graph lock to register cell %s", cell_id)+ with self.lock:+ LOGGER.debug("Acquired graph lock.")+ assert cell_id not in self.topology.cells++ # Add the cell to topology+ self.topology.add_node(cell_id, cell)++ # Initialize siblings for this cell+ self.siblings[cell_id] = set()++ # Process definitions and build sibling relationships FIRST+ # This must happen before computing edges because edge computation+ # needs to look up definitions+ for name, variable_data in cell.variable_data.items():+ siblings = self.definition_registry.register_definition(+ cell_id, name, variable_data+ )+ # Update siblings bidirectionally+ for sibling in siblings:+ self.siblings[cell_id].add(sibling)+ self.siblings[sibling].add(cell_id)++ # Now compute edges using EdgeComputer (which can now find the definitions)+ edge_computer = EdgeComputer(+ self.topology, self.definition_registry+ )+ parents, children = edge_computer.compute_edges_for_cell(+ cell_id, cell+ )++ # Add edges to topology+ for parent_id in parents:+ self.topology.add_edge(parent_id, cell_id)+ # Detect cycle for this edge+ self.cycle_tracker.detect_cycle_for_edge(+ (parent_id, cell_id), self.topology+ )++ for child_id in children:+ self.topology.add_edge(cell_id, child_id)+ # Detect cycle for this edge+ self.cycle_tracker.detect_cycle_for_edge(+ (cell_id, child_id), self.topology+ )++ LOGGER.debug("Registered cell %s and released graph lock", cell_id)+ if self.is_any_ancestor_stale(cell_id):+ self.set_stale({cell_id})++ if self.is_any_ancestor_disabled(cell_id):+ cell.set_runtime_state(status="disabled-transitively")++ def is_any_ancestor_stale(self, cell_id: CellId_t) -> bool:+ """Check if any ancestor of a cell is stale."""+ return any(+ self.topology.cells[cid].stale for cid in self.ancestors(cell_id)+ )++ def is_any_ancestor_disabled(self, cell_id: CellId_t) -> bool:+ """Check if any ancestor of a cell is disabled."""+ return any(+ self.topology.cells[cid].config.disabled+ for cid in self.ancestors(cell_id)+ )++ def disable_cell(self, cell_id: CellId_t) -> None:+ """Disables a cell in the graph.++ Does not mutate the graph (but does mutate cell statuses).++ Returns the ids of descendants that are disabled transitively.+ """+ if cell_id not in self.topology.cells:+ raise ValueError(f"Cell {cell_id} not found")++ from marimo._runtime.dataflow import transitive_closure++ for cid in transitive_closure(self, {cell_id}) - {cell_id}:+ cell = self.topology.cells[cid]+ cell.set_runtime_state(status="disabled-transitively")++ def enable_cell(self, cell_id: CellId_t) -> set[CellId_t]:+ """Enables a cell in the graph.++ Does not mutate the graph (but does mutate cell statuses).++ Returns:+ - set of cells that were stale and should be re-run+ """+ if cell_id not in self.topology.cells:+ raise ValueError(f"Cell {cell_id} not found")++ from marimo._runtime.dataflow import transitive_closure++ cells_to_run: set[CellId_t] = set()+ for cid in transitive_closure(self, {cell_id}):+ if not self.is_disabled(cid):+ child = self.topology.cells[cid]+ if child.stale:+ # cell was previously disabled, is no longer+ # disabled, and is stale: needs to run.+ cells_to_run.add(cid)+ if child.disabled_transitively:+ # cell is no longer disabled: status -> idle+ child.set_runtime_state("idle")+ return cells_to_run++ def delete_cell(self, cell_id: CellId_t) -> set[CellId_t]:+ """Removes a cell from the graph.++ Mutates the graph, acquiring `self.lock`.++ Returns the ids of the children of the removed cell.+ """+ LOGGER.debug("Acquiring graph lock to delete cell %s", cell_id)+ with self.lock:+ LOGGER.debug("Acquired graph lock to delete cell %s", cell_id)+ if cell_id not in self.topology.cells:+ raise ValueError(f"Cell {cell_id} not found")++ # Grab a reference to children before we remove it+ children = self.topology.children[cell_id].copy()++ # Removing this cell from its defs' definer sets+ cell = self.topology.cells[cell_id]+ self.definition_registry.unregister_definitions(cell_id, cell.defs)++ # Remove cycles that are broken from removing this cell+ edges = [+ (cell_id, child) for child in self.topology.children[cell_id]+ ] + [+ (parent, cell_id) for parent in self.topology.parents[cell_id]+ ]+ for e in edges:+ self.cycle_tracker.remove_cycles_with_edge(e)++ # Remove from siblings+ for sibling_id in self.siblings.get(cell_id, set()):+ self.siblings[sibling_id].discard(cell_id)+ self.siblings.pop(cell_id, None)++ # Purge this cell from the graph topology+ self.topology.remove_node(cell_id)++ LOGGER.debug("Deleted cell %s and Released graph lock.", cell_id)+ return children++ def is_disabled(self, cell_id: CellId_t) -> bool:+ """Check if a cell is disabled (directly or transitively)."""+ if cell_id not in self.topology.cells:+ raise ValueError(f"Cell {cell_id} not in graph.")+ cell = self.topology.cells[cell_id]+ if cell.config.disabled:+ return True+ seen: set[CellId_t] = set()+ queue = [cell_id]+ while queue:+ cid = queue.pop()+ seen.add(cid)+ for parent_id in self.topology.parents[cid]:+ if parent_id in seen:+ continue+ elif self.topology.cells[parent_id].config.disabled:+ return True+ else:+ queue.append(parent_id)+ return False++ def get_imports(+ self, cell_id: Optional[CellId_t] = None+ ) -> dict[Name, ImportData]:+ """Get imports from cell(s)."""+ imports = {}+ cells = (+ self.topology.cells.values()+ if cell_id is None+ else [self.topology.cells[cell_id]]+ )+ for cell in cells:+ for imported in cell.imports:+ imports[imported.definition] = imported+ return imports++ def get_multiply_defined(self) -> list[Name]:+ """Return a list of names that are defined in multiple cells."""+ return self.definition_registry.get_multiply_defined()++ def get_deleted_nonlocal_ref(self) -> list[Name]:+ """Get names that are deleted but defined elsewhere."""+ names: list[Name] = []+ for cell in self.topology.cells.values():+ for ref in cell.deleted_refs:+ if ref in self.definition_registry.definitions:+ names.append(ref)+ return names++ def set_stale(+ self, cell_ids: set[CellId_t], prune_imports: bool = False+ ) -> None:+ """Mark cells as stale (need re-execution)."""+ from marimo._runtime.dataflow import (+ get_import_block_relatives,+ transitive_closure,+ )++ relatives = (+ None if not prune_imports else get_import_block_relatives(self)+ )++ for cid in transitive_closure(self, cell_ids, relatives=relatives):+ self.topology.cells[cid].set_stale(stale=True)++ def get_stale(self) -> set[CellId_t]:+ """Get all stale cells."""+ return {cid for cid, cell in self.topology.cells.items() if cell.stale}++ def get_transitive_references(+ self,+ refs: set[Name],+ inclusive: bool = True,+ predicate: Callable[[Name, VariableData], bool] | None = None,+ ) -> set[Name]:+ """Return a set of the passed-in cells' references and their+ references on the block (function / class) level.++ If inclusive, includes the references of the passed-in cells in the+ set.++ If predicate, only references satisfying predicate(ref) are included+ """+ # TODO: Consider caching on the graph level and updating on register /+ # delete+ processed: set[Name] = set()+ queue: set[Name] = refs & self.definition_registry.definitions.keys()+ predicate = predicate or (lambda *_: True)++ while queue:+ # Should ideally be one cell per ref, but for completion, stay+ # agnostic to potenital cycles.+ cells = {+ cell_id+ for ref in queue+ for cell_id in self.definition_registry.definitions.get(+ ref, set()+ )+ }++ for cell_id in cells:+ data = self.topology.cells[cell_id].variable_data+ newly_processed = set(data.keys()) & queue+ processed.update(newly_processed)+ queue.difference_update(newly_processed)+ for variable in newly_processed:+ # variables can be defined multiple times in a single+ # cell ...+ for datum in data[variable]:+ if predicate(variable, datum):+ to_process = datum.required_refs - processed+ queue.update(+ to_process+ & self.definition_registry.definitions.keys()+ )+ # Private variables referenced by public functions+ # have to be included.+ processed.update(+ maybe_private+ for maybe_private in (+ to_process+ - self.definition_registry.definitions.keys()+ )+ if is_mangled_local(maybe_private, cell_id)+ )++ if inclusive:+ return processed | refs+ return processed - refs++ def copy(self, filename: None | str = None) -> DirectedGraph:+ """Return a deep copy of the graph by recompiling all cells.++ This is mainly useful in the case where recompilation must be done+ due to a dynamically changing notebook, where the line cache must be+ consistent with the cell code, e.g. for debugging.+ """+ from marimo._ast.compiler import compile_cell++ graph = DirectedGraph()+ with self.lock:+ for cid, old_cell in self.topology.cells.items():+ cell = compile_cell(+ old_cell.code,+ cell_id=cid,+ filename=filename,+ )+ # Carry over import data manually+ imported_defs = old_cell.import_workspace.imported_defs+ is_import_block = old_cell.import_workspace.is_import_block+ cell.import_workspace.imported_defs = imported_defs+ cell.import_workspace.is_import_block = is_import_block+ # Reregister+ graph.register_cell(cid, cell)+ return graph++ @property+ def cells(self) -> Mapping[CellId_t, CellImpl]:+ """Get the cells dictionary."""+ return self.topology.cells++ @property+ def parents(self) -> Mapping[CellId_t, set[CellId_t]]:+ """Get the parents dictionary."""+ return self.topology.parents++ @property+ def children(self) -> Mapping[CellId_t, set[CellId_t]]:+ """Get the children dictionary."""+ return self.topology.children++ def get_path(+ self, source: CellId_t, dst: CellId_t+ ) -> list[tuple[CellId_t, CellId_t]]:+ """Get a path from `source` to `dst`, if any."""+ return self.topology.get_path(source, dst)++ def descendants(self, cell_id: CellId_t) -> set[CellId_t]:+ """Get all descendants of a cell."""+ return self.topology.descendants(cell_id)++ def ancestors(self, cell_id: CellId_t) -> set[CellId_t]:+ """Get all ancestors of a cell."""+ return self.topology.ancestors(cell_id)++ @property+ def definitions(self) -> Mapping[Name, set[CellId_t]]:+ """Get the definitions dictionary."""+ return self.definition_registry.definitions++ @property+ def cycles(self) -> set[tuple[tuple[CellId_t, CellId_t], ...]]:+ """Get the cycles set."""+ return self.cycle_tracker.cyclesdiff --git a/marimo/_runtime/dataflow/runner.py b/marimo/_runtime/dataflow/runner.pynew file mode 100644index 0000000000..804c86deae--- /dev/null+++ b/marimo/_runtime/dataflow/runner.py@@ -0,0 +1,154 @@+# Copyright 2024 Marimo. All rights reserved.+"""Runner utility for executing individual cells in a graph."""++from __future__ import annotations++from typing import TYPE_CHECKING, Any++from marimo._runtime.executor import ExecutionConfig, get_executor++if TYPE_CHECKING:+ from marimo._ast.cell import CellImpl+ from marimo._runtime.dataflow.graph import DirectedGraph+ from marimo._types.ids import CellId_t+++class Runner:+ """Utility for running individual cells in a graph++ This class provides methods to a run a cell in the graph and obtain its+ output (last expression) and the values of its defs.++ If needed, the runner will recursively compute the values of the cell's+ refs by executing its ancestors. Refs can also be substituted by the+ caller.++ TODO(akshayka): Add an API for caching defs across cell runs.+ """++ def __init__(self, graph: DirectedGraph) -> None:+ self._graph = graph+ self._executor = get_executor(ExecutionConfig())++ @staticmethod+ def _returns(cell_impl: CellImpl, glbls: dict[str, Any]) -> dict[str, Any]:+ return {name: glbls[name] for name in cell_impl.defs if name in glbls}++ @staticmethod+ def _substitute_refs(+ cell_impl: CellImpl,+ glbls: dict[str, Any],+ kwargs: dict[str, Any],+ ) -> None:+ for argname, argvalue in kwargs.items():+ if argname in cell_impl.refs:+ glbls[argname] = argvalue+ else:+ raise ValueError(+ f"Cell got unexpected argument {argname}"+ f"The allowed arguments are {cell_impl.refs}."+ )++ def _get_ancestors(+ self, cell_impl: CellImpl, kwargs: dict[str, Any]+ ) -> set[CellId_t]:+ from marimo._runtime.dataflow import transitive_closure++ # Get the transitive closure of parents defining unsubstituted refs+ graph = self._graph+ substitutions = set(kwargs.keys())+ unsubstituted_refs = cell_impl.refs - substitutions+ parent_ids = set(+ [+ parent_id+ for parent_id in graph.parents[cell_impl.cell_id]+ if graph.cells[parent_id].defs.intersection(unsubstituted_refs)+ ]+ )+ return transitive_closure(graph, parent_ids, children=False)++ @staticmethod+ def _validate_kwargs(cell_impl: CellImpl, kwargs: dict[str, Any]) -> None:+ for argname in kwargs:+ if argname not in cell_impl.refs:+ raise ValueError(+ f"Cell got unexpected argument {argname}; "+ f"The allowed arguments are {cell_impl.refs}."+ )++ def is_coroutine(self, cell_id: CellId_t) -> bool:+ return self._graph.cells[cell_id].is_coroutine() or any(+ self._graph.cells[cid].is_coroutine()+ for cid in self._get_ancestors(+ self._graph.cells[cell_id], kwargs={}+ )+ )++ async def run_cell_async(+ self, cell_id: CellId_t, kwargs: dict[str, Any]+ ) -> tuple[Any, dict[str, Any]]:+ """Run a possibly async cell and its ancestors++ Substitutes kwargs as refs for the cell, omitting ancestors that+ whose refs are substituted.+ """+ from marimo._runtime.dataflow import topological_sort++ graph = self._graph+ cell_impl = graph.cells[cell_id]+ Runner._validate_kwargs(cell_impl, kwargs)+ ancestor_ids = self._get_ancestors(cell_impl, kwargs)++ glbls: dict[str, Any] = {}+ for cid in topological_sort(graph, ancestor_ids):+ await self._executor.execute_cell_async(+ graph.cells[cid], glbls, graph+ )++ Runner._substitute_refs(cell_impl, glbls, kwargs)+ output = await self._executor.execute_cell_async(+ graph.cells[cell_impl.cell_id], glbls, graph+ )+ defs = Runner._returns(cell_impl, glbls)+ return output, defs++ def run_cell_sync(+ self, cell_id: CellId_t, kwargs: dict[str, Any]+ ) -> tuple[Any, dict[str, Any]]:+ """Run a synchronous cell and its ancestors++ Substitutes kwargs as refs for the cell, omitting ancestors that+ whose refs are substituted.++ Raises a `RuntimeError` if the cell or any of its unsubstituted+ ancestors are coroutine functions.+ """+ from marimo._runtime.dataflow import topological_sort++ graph = self._graph+ cell_impl = graph.cells[cell_id]+ if cell_impl.is_coroutine():+ raise RuntimeError(+ "A coroutine function can't be run synchronously. "+ "Use `run_async()` instead"+ )++ Runner._validate_kwargs(cell_impl, kwargs)+ ancestor_ids = self._get_ancestors(cell_impl, kwargs)++ if any(graph.cells[cid].is_coroutine() for cid in ancestor_ids):+ raise RuntimeError(+ "Cell has an ancestor that is a "+ "coroutine (async) cell. Use `run_async()` instead"+ )++ glbls: dict[str, Any] = {}+ for cid in topological_sort(graph, ancestor_ids):+ self._executor.execute_cell(graph.cells[cid], glbls, graph)++ self._substitute_refs(cell_impl, glbls, kwargs)+ output = self._executor.execute_cell(+ graph.cells[cell_impl.cell_id], glbls, graph+ )+ defs = Runner._returns(cell_impl, glbls)+ return output, defsdiff --git a/marimo/_runtime/dataflow/topology.py b/marimo/_runtime/dataflow/topology.pynew file mode 100644index 0000000000..ff91164142--- /dev/null+++ b/marimo/_runtime/dataflow/topology.py@@ -0,0 +1,151 @@+# Copyright 2024 Marimo. All rights reserved.+"""Graph topology for cell dependencies.++This module provides the core graph structure for tracking cell relationships.+"""++from __future__ import annotations++from collections import deque+from dataclasses import dataclass, field+from typing import TYPE_CHECKING, Protocol++from marimo._runtime.dataflow.types import Edge++if TYPE_CHECKING:+ from collections.abc import Mapping++ from marimo._ast.cell import CellImpl+ from marimo._types.ids import CellId_t+++class GraphTopology(Protocol):+ """Graph topology protocol.++ This protocol is used to ensure that the graph topology is immutable. All mutations+ should go through the MutableGraphTopology class.+ """++ @property+ def cells(self) -> Mapping[CellId_t, CellImpl]: ...++ @property+ def children(self) -> Mapping[CellId_t, set[CellId_t]]: ...++ @property+ def parents(self) -> Mapping[CellId_t, set[CellId_t]]: ...++ def get_path(self, source: CellId_t, dst: CellId_t) -> list[Edge]: ...++ def ancestors(self, cell_id: CellId_t) -> set[CellId_t]: ...++ def descendants(self, cell_id: CellId_t) -> set[CellId_t]: ...+++@dataclass+class MutableGraphTopology(GraphTopology):+ """Pure graph structure: nodes and edges only.++ Responsibilities:+ - Store cells, parents, children mappings+ - Provide fast lookups and traversals+ - No business logic, just data structure+ """++ # Nodes in the graph+ _cells: dict[CellId_t, CellImpl] = field(default_factory=dict)++ # Edge (u, v) means v is a child of u, i.e., v has a reference+ # to something defined in u+ _children: dict[CellId_t, set[CellId_t]] = field(default_factory=dict)++ # Reversed edges (parent pointers) for convenience+ _parents: dict[CellId_t, set[CellId_t]] = field(default_factory=dict)++ @property+ def cells(self) -> Mapping[CellId_t, CellImpl]:+ return self._cells++ @property+ def children(self) -> Mapping[CellId_t, set[CellId_t]]:+ return self._children++ @property+ def parents(self) -> Mapping[CellId_t, set[CellId_t]]:+ return self._parents++ def ancestors(self, cell_id: CellId_t) -> set[CellId_t]:+ """Get all ancestors of a cell."""+ from marimo._runtime.dataflow import transitive_closure++ return transitive_closure(+ self, {cell_id}, children=False, inclusive=False+ )++ def descendants(self, cell_id: CellId_t) -> set[CellId_t]:+ """Get all descendants of a cell."""+ from marimo._runtime.dataflow import transitive_closure++ return transitive_closure(self, {cell_id}, inclusive=False)++ def add_node(self, cell_id: CellId_t, cell: CellImpl) -> None:+ """Add a cell to the graph topology."""+ assert cell_id not in self.cells, f"Cell {cell_id} already in graph"+ self._cells[cell_id] = cell+ self._children[cell_id] = set()+ self._parents[cell_id] = set()++ def remove_node(self, cell_id: CellId_t) -> None:+ """Remove a cell from the graph topology.++ Also removes all edges connected to this cell.+ """+ if cell_id not in self.cells:+ raise ValueError(f"Cell {cell_id} not found")++ # Remove from cells+ del self._cells[cell_id]+ del self._children[cell_id]+ del self._parents[cell_id]++ # Remove from other nodes' parent/child lists+ for elems in self.parents.values():+ if cell_id in elems:+ elems.remove(cell_id)+ for elems in self.children.values():+ if cell_id in elems:+ elems.remove(cell_id)++ def add_edge(self, parent: CellId_t, child: CellId_t) -> None:+ """Add an edge from parent to child."""+ self.children[parent].add(child)+ self.parents[child].add(parent)++ def remove_edge(self, parent: CellId_t, child: CellId_t) -> None:+ """Remove an edge from parent to child."""+ self.children[parent].discard(child)+ self.parents[child].discard(parent)++ def get_path(self, source: CellId_t, dst: CellId_t) -> list[Edge]:+ """Get a path from `source` to `dst`, if any.++ Returns an empty list if source == dst or if no path exists.+ """+ if source == dst:+ return []++ # BFS to find path+ # deque has O(1) append/pop operation+ queue: deque[tuple[CellId_t, list[Edge]]] = deque([(source, [])])+ found = {source} # set has O(1) lookups++ while queue:+ node, path = queue.popleft() # O(1) operation+ for cid in self.children[node]:+ if cid not in found:+ next_path = path + [(node, cid)]+ if cid == dst:+ return next_path+ found.add(cid)+ queue.append((cid, next_path))+ return []diff --git a/marimo/_runtime/dataflow/types.py b/marimo/_runtime/dataflow/types.pynew file mode 100644index 0000000000..863ac03113--- /dev/null+++ b/marimo/_runtime/dataflow/types.py@@ -0,0 +1,16 @@+# Copyright 2024 Marimo. All rights reserved.+"""Type definitions for dataflow graph."""++from __future__ import annotations++from marimo._types.ids import CellId_t++# Edge is a tuple of (parent, child) cell IDs+Edge = tuple[CellId_t, CellId_t]++# EdgeWithVar uses a tuple (not a set) for the variables linking the cells+# as sets are not JSON-serializable (required by static_notebook_template()).+# The first entry is the source node; the second entry is a tuple of defs from+# the source read by the destination; and the third entry is the destination+# node.+EdgeWithVar = tuple[CellId_t, tuple[str, ...], CellId_t]diff --git a/marimo/_runtime/packages/module_registry.py b/marimo/_runtime/packages/module_registry.pyindex 9c41a301ac..6c24dfe010 100644--- a/marimo/_runtime/packages/module_registry.py+++ b/marimo/_runtime/packages/module_registry.py@@ -4,7 +4,7 @@ from __future__ import annotationsimport importlib.utilimport sys-from marimo._runtime.dataflow import DirectedGraph+from marimo._runtime.dataflow.topology import GraphTopologyfrom marimo._types.ids import CellId_t@@ -26,7 +26,7 @@ def _is_module_installed(module_name: str) -> bool:class ModuleRegistry:def __init__(- self, graph: DirectedGraph, excluded_modules: set[str] | None = None+ self, graph: GraphTopology, excluded_modules: set[str] | None = None) -> None:self.graph = graph# modules that do not have corresponding packages on package indexdiff --git a/marimo/_runtime/reload/module_watcher.py b/marimo/_runtime/reload/module_watcher.pyindex a8fd412722..d697ccde19 100644--- a/marimo/_runtime/reload/module_watcher.py+++ b/marimo/_runtime/reload/module_watcher.py@@ -205,7 +205,7 @@ def watch_modules(modname_to_cell_id[modname]for modname in stale_modules),- relatives=dataflow.import_block_relatives,+ relatives=dataflow.get_import_block_relatives(graph),)for cid in stale_cell_ids:graph.cells[cid].set_stale(stale=True, stream=stream)diff --git a/marimo/_runtime/runner/cell_runner.py b/marimo/_runtime/runner/cell_runner.pyindex 341a00175e..98b0cfd619 100644--- a/marimo/_runtime/runner/cell_runner.py+++ b/marimo/_runtime/runner/cell_runner.py@@ -195,10 +195,13 @@ class Runner:predicate=lambda cell: cell.stale,))+if execution_mode == "autorun":# in autorun/eager mode, descendants are also run;cells_to_run = dataflow.transitive_closure(- graph, cells_to_run, relatives=dataflow.import_block_relatives+ graph,+ cells_to_run,+ relatives=dataflow.get_import_block_relatives(graph),)return dataflow.topological_sort(@@ -643,7 +646,7 @@ class Runner:blamed_cell = Nonetry:(blamed_cell, *_) = self.graph.get_defining_cells(ref)- except KeyError:+ except (KeyError, ValueError):# The reference is not found anywhere else in the graph# but it might be privateref, var_cell_id = unmangle_local(ref)diff --git a/marimo/_runtime/runner/hooks_preparation.py b/marimo/_runtime/runner/hooks_preparation.pyindex 9eebb30cdf..dddd28948b 100644--- a/marimo/_runtime/runner/hooks_preparation.py+++ b/marimo/_runtime/runner/hooks_preparation.py@@ -4,7 +4,6 @@ from __future__ import annotationsfrom typing import Callablefrom marimo._runtime import dataflow-from marimo._runtime.dataflow import import_block_relativesfrom marimo._runtime.runner import cell_runnerfrom marimo._tracer import kernel_tracer@@ -20,7 +19,7 @@ def _update_stale_statuses(runner: cell_runner.Runner) -> None:graph,set(runner.cells_to_run),inclusive=False,- relatives=import_block_relatives,+ relatives=dataflow.get_import_block_relatives(graph),):graph.cells[cid].set_stale(stale=True)diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.pyindex 640f9de3f2..5dc1dd4481 100644--- a/marimo/_runtime/runtime.py+++ b/marimo/_runtime/runtime.py@@ -1743,7 +1743,7 @@ class Kernel:dataflow.transitive_closure(self.graph,cells_to_run,- relatives=dataflow.import_block_relatives,+ relatives=dataflow.get_import_block_relatives(self.graph),))if self.module_watcher is not None:diff --git a/marimo/_save/hash.py b/marimo/_save/hash.pyindex 65dcc8f46b..9cf696de01 100644--- a/marimo/_save/hash.py+++ b/marimo/_save/hash.py@@ -22,6 +22,7 @@ from marimo._dependencies.dependencies import DependencyManagerfrom marimo._plugins.ui._core.ui_element import UIElementfrom marimo._runtime.context import ContextNotInitializedError, get_contextfrom marimo._runtime.dataflow import induced_subgraph+from marimo._runtime.dataflow.topology import GraphTopologyfrom marimo._runtime.primitives import (CLONE_PRIMITIVES,FN_CACHE_TYPE,@@ -146,7 +147,7 @@ def hash_function(def hash_cell_group(cell_ids: set[CellId_t],- graph: DirectedGraph,+ graph: GraphTopology,hash_type: str = DEFAULT_HASH,) -> bytes:hash_alg = hashlib.new(hash_type, usedforsecurity=False)@@ -162,7 +163,7 @@ def hash_cell_group(def hash_cell_execution(- cell_id: CellId_t, graph: DirectedGraph, hash_type: str = DEFAULT_HASH+ cell_id: CellId_t, graph: GraphTopology, hash_type: str = DEFAULT_HASH) -> bytes:ancestors = graph.ancestors(cell_id)return hash_cell_group(ancestors, graph, hash_type)diff --git a/tests/_lint/test_validate_graph.py b/tests/_lint/test_validate_graph.pyindex 53ef340779..41cb8db68a 100644--- a/tests/_lint/test_validate_graph.py+++ b/tests/_lint/test_validate_graph.py@@ -75,12 +75,16 @@ def test_two_node_cycle() -> None:graph.register_cell("1", parse_cell("y = x"))errors = check_for_errors(graph)assert set(errors.keys()) == set(["0", "1"])- assert errors["0"] == (- CycleError(edges_with_vars=(("0", ("x",), "1"), ("1", ("y",), "0"))),- )- assert errors["1"] == (+ # Edge ordering in returned error is not deterministic, so we list both+ # possible cycles+ expected_cycle = [CycleError(edges_with_vars=(("0", ("x",), "1"), ("1", ("y",), "0"))),- )+ CycleError(edges_with_vars=(("1", ("y",), "0"), ("0", ("x",), "1"))),+ ]+ assert len(errors["0"]) == 1+ assert len(errors["1"]) == 1+ assert errors["0"][0] in expected_cycle+ assert errors["1"][0] in expected_cycledef test_three_node_cycle() -> None:diff --git a/tests/_runtime/test_dataflow.py b/tests/_runtime/test_dataflow.pyindex d8eab1ffb5..4df98513d8 100644--- a/tests/_runtime/test_dataflow.py+++ b/tests/_runtime/test_dataflow.py@@ -1204,7 +1204,7 @@ def test_import_block_relatives() -> None:graph.register_cell("3", fourth_cell)# Test the function- children = dataflow.import_block_relatives(graph, "0", True)+ children = dataflow.get_import_block_relatives(graph)("0", True)# Should include cells that use pd and npassert "1" in children@@ -1212,7 +1212,7 @@ def test_import_block_relatives() -> None:assert "3" not in children# Test with a non-import block- children = dataflow.import_block_relatives(graph, "3", True)+ children = dataflow.get_import_block_relatives(graph)("3", True)# Should just return normal childrenassert children == graph.children["3"]diff --git a/tests/_runtime/test_edges.py b/tests/_runtime/test_edges.pynew file mode 100644index 0000000000..2dd8dd897f--- /dev/null+++ b/tests/_runtime/test_edges.py@@ -0,0 +1,564 @@+# Copyright 2024 Marimo. All rights reserved.+from __future__ import annotations++from functools import partial++import pytest++from marimo._ast import compiler+from marimo._dependencies.dependencies import DependencyManager+from marimo._runtime.dataflow.definitions import DefinitionRegistry+from marimo._runtime.dataflow.edges import EdgeComputer+from marimo._runtime.dataflow.topology import MutableGraphTopology++parse_cell = partial(compiler.compile_cell, cell_id="0")++HAS_SQL = DependencyManager.duckdb.has() and DependencyManager.polars.has()+++class TestEdgeComputer:+ """Tests for EdgeComputer class."""++ def setup_method(self) -> None:+ """Set up a fresh topology and definitions for each test."""+ self.topology = MutableGraphTopology()+ self.definitions = DefinitionRegistry()+ self.edge_computer = EdgeComputer(self.topology, self.definitions)++ def test_get_referring_cells_python_no_refs(self) -> None:+ """Test getting referring cells when no cells reference the variable."""+ cell1 = parse_cell("x = 1")+ self.topology.add_node("cell_1", cell1)++ referring = self.edge_computer.get_referring_cells(+ "x", language="python"+ )++ assert referring == set()++ def test_get_referring_cells_python_single_ref(self) -> None:+ """Test getting referring cells with a single reference."""+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)++ referring = self.edge_computer.get_referring_cells(+ "x", language="python"+ )++ assert referring == {"cell_2"}++ def test_get_referring_cells_python_multiple_refs(self) -> None:+ """Test getting referring cells with multiple references."""+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")+ cell3 = parse_cell("z = x * 2")+ cell4 = parse_cell("w = x + y")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)+ self.topology.add_node("cell_3", cell3)+ self.topology.add_node("cell_4", cell4)++ referring = self.edge_computer.get_referring_cells(+ "x", language="python"+ )++ assert referring == {"cell_2", "cell_3", "cell_4"}++ def test_get_referring_cells_python_no_self_reference(self) -> None:+ """Test that a cell doesn't include itself in referring cells."""+ cell1 = parse_cell("x = x + 1")++ self.topology.add_node("cell_1", cell1)++ referring = self.edge_computer.get_referring_cells(+ "x", language="python"+ )++ # cell_1 both defines and references x, but should be in the result+ assert referring == {"cell_1"}++ def test_compute_edges_simple_dependency(self) -> None:+ """Test computing edges for a simple dependency."""+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)++ # Register cell1's definitions+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)++ # Compute edges for cell2+ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_2", cell2+ )++ assert parents == {"cell_1"}+ assert children == set()++ def test_compute_edges_multiple_parents(self) -> None:+ """Test computing edges with multiple parent dependencies."""+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = 2")+ cell3 = parse_cell("z = x + y")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)+ self.topology.add_node("cell_3", cell3)++ # Register definitions+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)+ for name, var_data in cell2.variable_data.items():+ self.definitions.register_definition("cell_2", name, var_data)++ # Compute edges for cell3+ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_3", cell3+ )++ assert parents == {"cell_1", "cell_2"}+ assert children == set()++ def test_compute_edges_with_children(self) -> None:+ """Test computing edges when a cell has both parents and children."""+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")+ cell3 = parse_cell("z = y")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)+ self.topology.add_node("cell_3", cell3)++ # Register definitions for cell1+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)++ # Register definitions for cell2+ for name, var_data in cell2.variable_data.items():+ self.definitions.register_definition("cell_2", name, var_data)++ # Compute edges for cell2 (middle cell)+ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_2", cell2+ )++ assert parents == {"cell_1"}+ assert children == {"cell_3"}++ def test_compute_edges_no_dependencies(self) -> None:+ """Test computing edges for an independent cell."""+ cell1 = parse_cell("x = 1")++ self.topology.add_node("cell_1", cell1)++ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_1", cell1+ )++ assert parents == set()+ assert children == set()++ def test_compute_edges_with_deleted_refs(self) -> None:+ """Test computing edges when a cell deletes a variable."""+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")+ cell3 = parse_cell("del x")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)+ self.topology.add_node("cell_3", cell3)++ # Register definitions+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)++ # Compute edges for cell3 (which deletes x)+ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_3", cell3+ )++ # cell3 should become a child of cells that reference x (cell2)+ assert parents == {"cell_1", "cell_2"}+ assert children == set()++ def test_compute_edges_multiple_deleted_refs_create_cycle(self) -> None:+ """Test that multiple cells deleting the same variable create cycles."""+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("del x")+ cell3 = parse_cell("del x")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)+ self.topology.add_node("cell_3", cell3)++ # Register definitions+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)++ # Compute edges for cell2+ parents2, children2 = self.edge_computer.compute_edges_for_cell(+ "cell_2", cell2+ )++ # Compute edges for cell3+ parents3, children3 = self.edge_computer.compute_edges_for_cell(+ "cell_3", cell3+ )++ # Both cells should have cell1 as parent+ assert "cell_1" in parents2+ assert "cell_1" in parents3++ # cell2 and cell3 should be children of each other (cycle)+ assert "cell_3" in children2+ assert "cell_2" in parents3++ def test_compute_edges_cell_references_its_own_def(self) -> None:+ """Test computing edges when a cell references its own definition."""+ cell1 = parse_cell("x = x + 1 if 'x' in globals() else 0")++ self.topology.add_node("cell_1", cell1)++ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_1", cell1+ )++ # A cell doesn't create an edge to itself+ assert parents == set()+ assert children == set()++ def test_compute_edges_diamond_dependency(self) -> None:+ """Test computing edges in a diamond-shaped dependency graph."""+ cell1 = parse_cell("a = 1")+ cell2 = parse_cell("b = a")+ cell3 = parse_cell("c = a")+ cell4 = parse_cell("d = b + c")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)+ self.topology.add_node("cell_3", cell3)+ self.topology.add_node("cell_4", cell4)++ # Register definitions+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)+ for name, var_data in cell2.variable_data.items():+ self.definitions.register_definition("cell_2", name, var_data)+ for name, var_data in cell3.variable_data.items():+ self.definitions.register_definition("cell_3", name, var_data)++ # Check cell1's children (only direct children, not grandchildren)+ parents1, children1 = self.edge_computer.compute_edges_for_cell(+ "cell_1", cell1+ )+ assert children1 == {"cell_2", "cell_3"}++ # Check cell4's parents (it references b and c, not a directly)+ parents4, children4 = self.edge_computer.compute_edges_for_cell(+ "cell_4", cell4+ )+ assert parents4 == {"cell_2", "cell_3"}++ def test_compute_edges_with_undefined_reference(self) -> None:+ """Test computing edges when a cell references an undefined variable."""+ cell1 = parse_cell("y = x") # x is not defined++ self.topology.add_node("cell_1", cell1)++ # Should not raise an error, just return empty parents+ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_1", cell1+ )++ assert parents == set()+ assert children == set()++ def test_compute_edges_with_function_closure(self) -> None:+ """Test computing edges with function closures."""+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("def foo():\n return x")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)++ # Register definitions+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)++ # Compute edges for cell2+ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_2", cell2+ )++ assert parents == {"cell_1"}+ assert children == set()++ def test_compute_edges_with_class_definition(self) -> None:+ """Test computing edges with class definitions."""+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("class MyClass:\n value = x")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)++ # Register definitions+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)++ # Compute edges for cell2+ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_2", cell2+ )++ assert parents == {"cell_1"}+ assert children == set()++ def test_compute_edges_chain_of_dependencies(self) -> None:+ """Test a long chain of dependencies."""+ cells = []+ for i in range(5):+ if i == 0:+ cell = parse_cell(f"x{i} = {i}")+ else:+ cell = parse_cell(f"x{i} = x{i - 1} + 1")+ self.topology.add_node(f"cell_{i}", cell)+ cells.append((f"cell_{i}", cell))++ # Register definitions+ for cell_id, cell in cells[:-1]:+ for name, var_data in cell.variable_data.items():+ self.definitions.register_definition(cell_id, name, var_data)++ # Check last cell's parents+ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_4", cells[-1][1]+ )++ assert parents == {"cell_3"}+ assert children == set()++ @pytest.mark.skipif(not HAS_SQL, reason="requires duckdb and polars")+ def test_get_referring_cells_sql_basic(self) -> None:+ """Test getting referring cells for SQL variables."""+ # SQL cell that creates a table+ cell1 = compiler.compile_cell(+ "import duckdb; import polars as pl; df = pl.DataFrame({'a': [1, 2, 3]})",+ cell_id="0",+ )+ cell2 = compiler.compile_cell("df.select(pl.col('a'))", cell_id="1")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)++ referring = self.edge_computer.get_referring_cells(+ "df", language="python"+ )++ assert referring == {"cell_2"}++ @pytest.mark.skipif(not HAS_SQL, reason="requires duckdb and polars")+ def test_compute_edges_sql_to_python(self) -> None:+ """Test that SQL table definitions don't create edges to Python refs."""+ # This tests the language isolation: SQL defs don't leak to Python+ cell1 = compiler.compile_cell(+ "df = mo.sql('CREATE TABLE my_table AS SELECT 1 as x')",+ cell_id="0",+ )+ cell2 = compiler.compile_cell("print(my_table)", cell_id="1")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)++ # Register SQL table definition+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)++ # Compute edges for cell2 (Python cell)+ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_2", cell2+ )++ # SQL definitions should not create edges to Python cells+ # (unless explicitly exported)+ # Since my_table is SQL, it won't be a parent+ # However, the compiler might not recognize it as SQL without execution+ # So this test verifies the edge computation logic++ def test_compute_edges_multiple_definitions_of_same_var(self) -> None:+ """Test computing edges when multiple cells define the same variable."""+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("x = 2")+ cell3 = parse_cell("y = x")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)+ self.topology.add_node("cell_3", cell3)++ # Register both definitions of x+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)+ for name, var_data in cell2.variable_data.items():+ self.definitions.register_definition("cell_2", name, var_data)++ # Compute edges for cell3+ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_3", cell3+ )++ # cell3 should depend on both cells that define x+ assert parents == {"cell_1", "cell_2"}++ def test_compute_edges_complex_scenario(self) -> None:+ """Test a complex scenario with multiple patterns."""+ # Create a complex graph:+ # cell1: defines a+ # cell2: defines b = a+ # cell3: defines c = a + b+ # cell4: uses c, deletes a+ # cell5: defines d = b++ cell1 = parse_cell("a = 1")+ cell2 = parse_cell("b = a")+ cell3 = parse_cell("c = a + b")+ cell4 = parse_cell("result = c; del a")+ cell5 = parse_cell("d = b")++ cells = [+ ("cell_1", cell1),+ ("cell_2", cell2),+ ("cell_3", cell3),+ ("cell_4", cell4),+ ("cell_5", cell5),+ ]++ for cell_id, cell in cells:+ self.topology.add_node(cell_id, cell)++ # Register definitions for cells 1-3+ for cell_id, cell in cells[:3]:+ for name, var_data in cell.variable_data.items():+ self.definitions.register_definition(cell_id, name, var_data)++ # Compute edges for cell4+ parents4, children4 = self.edge_computer.compute_edges_for_cell(+ "cell_4", cell4+ )++ # cell4 should:+ # - depend on cell3 (for c)+ # - depend on cell1 (for del a)+ # - depend on cell2 (because it references a, which cell4 deletes)+ assert "cell_3" in parents4 # uses c+ assert "cell_1" in parents4 # deletes a+ assert "cell_2" in parents4 # cell2 references a++ def test_is_valid_cell_reference_missing_cell(self) -> None:+ """Test validation when a cell reference points to a non-existent cell."""+ # The _is_valid_cell_reference method logs an error when a cell is not found+ result = self.edge_computer._is_valid_cell_reference(+ "nonexistent", "some_var"+ )++ assert result is False++ def test_is_valid_cell_reference_valid_cell(self) -> None:+ """Test validation when a cell reference is valid."""+ cell1 = parse_cell("x = 1")+ self.topology.add_node("cell_1", cell1)++ result = self.edge_computer._is_valid_cell_reference("cell_1", "x")++ assert result is True++ def test_compute_edges_with_imports(self) -> None:+ """Test computing edges with import statements."""+ cell1 = parse_cell("import math")+ cell2 = parse_cell("x = math.pi")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)++ # Register definitions+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)++ # Compute edges for cell2+ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_2", cell2+ )++ assert parents == {"cell_1"}++ def test_compute_edges_with_from_import(self) -> None:+ """Test computing edges with from-import statements."""+ cell1 = parse_cell("from math import pi")+ cell2 = parse_cell("x = pi")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)++ # Register definitions+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)++ # Compute edges for cell2+ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_2", cell2+ )++ assert parents == {"cell_1"}++ def test_compute_edges_list_comprehension(self) -> None:+ """Test computing edges with list comprehensions."""+ cell1 = parse_cell("numbers = [1, 2, 3]")+ cell2 = parse_cell("squared = [x**2 for x in numbers]")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)++ # Register definitions+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)++ # Compute edges for cell2+ parents, children = self.edge_computer.compute_edges_for_cell(+ "cell_2", cell2+ )++ assert parents == {"cell_1"}++ def test_compute_edges_with_multiple_assignments(self) -> None:+ """Test computing edges with multiple assignments."""+ cell1 = parse_cell("x = y = z = 1")+ cell2 = parse_cell("a = x")+ cell3 = parse_cell("b = y")+ cell4 = parse_cell("c = z")++ self.topology.add_node("cell_1", cell1)+ self.topology.add_node("cell_2", cell2)+ self.topology.add_node("cell_3", cell3)+ self.topology.add_node("cell_4", cell4)++ # Register definitions+ for name, var_data in cell1.variable_data.items():+ self.definitions.register_definition("cell_1", name, var_data)++ # All cells should depend on cell1+ for cell_id in ["cell_2", "cell_3", "cell_4"]:+ cell = self.topology.cells[cell_id]+ parents, children = self.edge_computer.compute_edges_for_cell(+ cell_id, cell+ )+ assert parents == {"cell_1"}++ # Check that cell1 has all three as children+ parents1, children1 = self.edge_computer.compute_edges_for_cell(+ "cell_1", cell1+ )+ assert children1 == {"cell_2", "cell_3", "cell_4"}diff --git a/tests/_runtime/test_topology.py b/tests/_runtime/test_topology.pynew file mode 100644index 0000000000..e620381a27--- /dev/null+++ b/tests/_runtime/test_topology.py@@ -0,0 +1,429 @@+# Copyright 2024 Marimo. All rights reserved.+from __future__ import annotations++from functools import partial++import pytest++from marimo._ast import compiler+from marimo._runtime.dataflow.topology import MutableGraphTopology+from marimo._types.ids import CellId_t++parse_cell = partial(compiler.compile_cell, cell_id=CellId_t("0"))+++class TestMutableGraphTopology:+ """Tests for MutableGraphTopology class."""++ def test_empty_graph(self) -> None:+ """Test that an empty graph has no cells, parents, or children."""+ graph = MutableGraphTopology()+ assert len(graph.cells) == 0+ assert len(graph.children) == 0+ assert len(graph.parents) == 0++ def test_add_single_node(self) -> None:+ """Test adding a single node to the graph."""+ graph = MutableGraphTopology()+ cell = parse_cell("x = 1")++ graph.add_node("cell_1", cell)++ assert "cell_1" in graph.cells+ assert graph.cells["cell_1"] == cell+ assert graph.children["cell_1"] == set()+ assert graph.parents["cell_1"] == set()++ def test_add_duplicate_node_raises_error(self) -> None:+ """Test that adding a duplicate node raises an assertion error."""+ graph = MutableGraphTopology()+ cell = parse_cell("x = 1")++ graph.add_node("cell_1", cell)++ with pytest.raises(+ AssertionError, match="Cell cell_1 already in graph"+ ):+ graph.add_node("cell_1", cell)++ def test_add_multiple_nodes(self) -> None:+ """Test adding multiple nodes to the graph."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = 2")+ cell3 = parse_cell("z = 3")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_node("cell_3", cell3)++ assert len(graph.cells) == 3+ assert "cell_1" in graph.cells+ assert "cell_2" in graph.cells+ assert "cell_3" in graph.cells++ def test_remove_node(self) -> None:+ """Test removing a node from the graph."""+ graph = MutableGraphTopology()+ cell = parse_cell("x = 1")++ graph.add_node("cell_1", cell)+ graph.remove_node("cell_1")++ assert "cell_1" not in graph.cells+ assert "cell_1" not in graph.children+ assert "cell_1" not in graph.parents++ def test_remove_nonexistent_node_raises_error(self) -> None:+ """Test that removing a non-existent node raises ValueError."""+ graph = MutableGraphTopology()++ with pytest.raises(ValueError, match="Cell cell_1 not found"):+ graph.remove_node("cell_1")++ def test_remove_node_with_edges(self) -> None:+ """Test that removing a node also removes its edges."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")+ cell3 = parse_cell("z = y")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_node("cell_3", cell3)+ graph.add_edge("cell_1", "cell_2")+ graph.add_edge("cell_2", "cell_3")++ # Remove middle node+ graph.remove_node("cell_2")++ # Check that cell_2 is removed from children and parents+ assert "cell_2" not in graph.cells+ assert "cell_2" not in graph.children["cell_1"]+ assert "cell_2" not in graph.parents["cell_3"]++ def test_add_edge(self) -> None:+ """Test adding an edge between two nodes."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_edge("cell_1", "cell_2")++ assert "cell_2" in graph.children["cell_1"]+ assert "cell_1" in graph.parents["cell_2"]++ def test_add_multiple_edges(self) -> None:+ """Test adding multiple edges in the graph."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")+ cell3 = parse_cell("z = x + y")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_node("cell_3", cell3)+ graph.add_edge("cell_1", "cell_2")+ graph.add_edge("cell_1", "cell_3")+ graph.add_edge("cell_2", "cell_3")++ # cell_1 has two children+ assert graph.children["cell_1"] == {"cell_2", "cell_3"}+ # cell_2 has one child+ assert graph.children["cell_2"] == {"cell_3"}+ # cell_3 has two parents+ assert graph.parents["cell_3"] == {"cell_1", "cell_2"}++ def test_remove_edge(self) -> None:+ """Test removing an edge between two nodes."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_edge("cell_1", "cell_2")+ graph.remove_edge("cell_1", "cell_2")++ assert "cell_2" not in graph.children["cell_1"]+ assert "cell_1" not in graph.parents["cell_2"]++ def test_remove_nonexistent_edge(self) -> None:+ """Test that removing a non-existent edge doesn't raise error."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = 2")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)++ # Should not raise an error (using discard)+ graph.remove_edge("cell_1", "cell_2")++ assert "cell_2" not in graph.children["cell_1"]+ assert "cell_1" not in graph.parents["cell_2"]++ def test_get_path_direct_edge(self) -> None:+ """Test getting a path between two directly connected nodes."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_edge("cell_1", "cell_2")++ path = graph.get_path("cell_1", "cell_2")++ assert path == [("cell_1", "cell_2")]++ def test_get_path_indirect(self) -> None:+ """Test getting a path through intermediate nodes."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")+ cell3 = parse_cell("z = y")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_node("cell_3", cell3)+ graph.add_edge("cell_1", "cell_2")+ graph.add_edge("cell_2", "cell_3")++ path = graph.get_path("cell_1", "cell_3")++ assert path == [("cell_1", "cell_2"), ("cell_2", "cell_3")]++ def test_get_path_no_path(self) -> None:+ """Test getting a path when no path exists."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = 2")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)++ path = graph.get_path("cell_1", "cell_2")++ assert path == []++ def test_get_path_same_node(self) -> None:+ """Test getting a path from a node to itself."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")++ graph.add_node("cell_1", cell1)++ path = graph.get_path("cell_1", "cell_1")++ assert path == []++ def test_get_path_complex_graph(self) -> None:+ """Test getting a path in a more complex graph with multiple paths."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("a = 1")+ cell2 = parse_cell("b = a")+ cell3 = parse_cell("c = a")+ cell4 = parse_cell("d = b + c")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_node("cell_3", cell3)+ graph.add_node("cell_4", cell4)+ graph.add_edge("cell_1", "cell_2")+ graph.add_edge("cell_1", "cell_3")+ graph.add_edge("cell_2", "cell_4")+ graph.add_edge("cell_3", "cell_4")++ # BFS should find one of the shortest paths+ path = graph.get_path("cell_1", "cell_4")++ # Should be length 2 (one of: cell_1->cell_2->cell_4 or cell_1->cell_3->cell_4)+ assert len(path) == 2+ assert path[0][0] == "cell_1"+ assert path[1][1] == "cell_4"++ def test_ancestors_empty(self) -> None:+ """Test ancestors of a node with no parents."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")++ graph.add_node("cell_1", cell1)++ ancestors = graph.ancestors("cell_1")++ assert ancestors == set()++ def test_ancestors_single_parent(self) -> None:+ """Test ancestors with a single parent."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_edge("cell_1", "cell_2")++ ancestors = graph.ancestors("cell_2")++ assert ancestors == {"cell_1"}++ def test_ancestors_multiple_generations(self) -> None:+ """Test ancestors across multiple generations."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")+ cell3 = parse_cell("z = y")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_node("cell_3", cell3)+ graph.add_edge("cell_1", "cell_2")+ graph.add_edge("cell_2", "cell_3")++ ancestors = graph.ancestors("cell_3")++ assert ancestors == {"cell_1", "cell_2"}++ def test_ancestors_diamond_shape(self) -> None:+ """Test ancestors in a diamond-shaped graph."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("a = 1")+ cell2 = parse_cell("b = a")+ cell3 = parse_cell("c = a")+ cell4 = parse_cell("d = b + c")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_node("cell_3", cell3)+ graph.add_node("cell_4", cell4)+ graph.add_edge("cell_1", "cell_2")+ graph.add_edge("cell_1", "cell_3")+ graph.add_edge("cell_2", "cell_4")+ graph.add_edge("cell_3", "cell_4")++ ancestors = graph.ancestors("cell_4")++ assert ancestors == {"cell_1", "cell_2", "cell_3"}++ def test_descendants_empty(self) -> None:+ """Test descendants of a node with no children."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")++ graph.add_node("cell_1", cell1)++ descendants = graph.descendants("cell_1")++ assert descendants == set()++ def test_descendants_single_child(self) -> None:+ """Test descendants with a single child."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_edge("cell_1", "cell_2")++ descendants = graph.descendants("cell_1")++ assert descendants == {"cell_2"}++ def test_descendants_multiple_generations(self) -> None:+ """Test descendants across multiple generations."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")+ cell2 = parse_cell("y = x")+ cell3 = parse_cell("z = y")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_node("cell_3", cell3)+ graph.add_edge("cell_1", "cell_2")+ graph.add_edge("cell_2", "cell_3")++ descendants = graph.descendants("cell_1")++ assert descendants == {"cell_2", "cell_3"}++ def test_descendants_diamond_shape(self) -> None:+ """Test descendants in a diamond-shaped graph."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("a = 1")+ cell2 = parse_cell("b = a")+ cell3 = parse_cell("c = a")+ cell4 = parse_cell("d = b + c")++ graph.add_node("cell_1", cell1)+ graph.add_node("cell_2", cell2)+ graph.add_node("cell_3", cell3)+ graph.add_node("cell_4", cell4)+ graph.add_edge("cell_1", "cell_2")+ graph.add_edge("cell_1", "cell_3")+ graph.add_edge("cell_2", "cell_4")+ graph.add_edge("cell_3", "cell_4")++ descendants = graph.descendants("cell_1")++ assert descendants == {"cell_2", "cell_3", "cell_4"}++ def test_properties_are_readonly(self) -> None:+ """Test that the properties return Mapping (read-only) types."""+ graph = MutableGraphTopology()+ cell1 = parse_cell("x = 1")++ graph.add_node("cell_1", cell1)++ # The properties should return Mapping types+ from collections.abc import Mapping++ assert isinstance(graph.cells, Mapping)+ assert isinstance(graph.children, Mapping)+ assert isinstance(graph.parents, Mapping)++ def test_complex_graph_operations(self) -> None:+ """Test a complex sequence of operations on the graph."""+ graph = MutableGraphTopology()++ # Build a graph: 1 -> 2 -> 3 -> 4+ # \-> 5 -> 6+ cells = {f"cell_{i}": parse_cell(f"x{i} = {i}") for i in range(1, 7)}++ for cell_id, cell in cells.items():+ graph.add_node(cell_id, cell)++ graph.add_edge("cell_1", "cell_2")+ graph.add_edge("cell_2", "cell_3")+ graph.add_edge("cell_3", "cell_4")+ graph.add_edge("cell_2", "cell_5")+ graph.add_edge("cell_5", "cell_6")++ # Test various operations+ assert graph.descendants("cell_1") == {+ "cell_2",+ "cell_3",+ "cell_4",+ "cell_5",+ "cell_6",+ }+ assert graph.descendants("cell_2") == {+ "cell_3",+ "cell_4",+ "cell_5",+ "cell_6",+ }+ assert graph.ancestors("cell_4") == {"cell_1", "cell_2", "cell_3"}+ assert graph.ancestors("cell_6") == {"cell_1", "cell_2", "cell_5"}++ # Remove an edge and test again+ graph.remove_edge("cell_2", "cell_5")+ assert graph.descendants("cell_2") == {"cell_3", "cell_4"}++ # Remove a node and test+ graph.remove_node("cell_3")+ assert graph.descendants("cell_1") == {"cell_2"}+ assert "cell_3" not in graph.cells