Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

refactor(runtime): split kernel command dispatch into router + callback bundles (#9577)

Myles Scolnickc09cb3d81b6c
diff --git a/marimo/_runtime/callbacks/__init__.py b/marimo/_runtime/callbacks/__init__.py
new file mode 100644
index 0000000000..95b8ffe7be
--- /dev/null
+++ b/marimo/_runtime/callbacks/__init__.py
@@ -0,0 +1,22 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from marimo._runtime.callbacks.cache import CacheCallbacks
+from marimo._runtime.callbacks.datasets import DatasetCallbacks
+from marimo._runtime.callbacks.external_storage import (
+ ExternalStorageCallbacks,
+)
+from marimo._runtime.callbacks.packages import PackagesCallbacks
+from marimo._runtime.callbacks.protocol import KernelCallback
+from marimo._runtime.callbacks.secrets import SecretsCallbacks
+from marimo._runtime.callbacks.sql import SqlCallbacks
+
+__all__ = [
+ "CacheCallbacks",
+ "DatasetCallbacks",
+ "ExternalStorageCallbacks",
+ "KernelCallback",
+ "PackagesCallbacks",
+ "SecretsCallbacks",
+ "SqlCallbacks",
+]
diff --git a/marimo/_runtime/callbacks/cache.py b/marimo/_runtime/callbacks/cache.py
new file mode 100644
index 0000000000..ceefc4149c
--- /dev/null
+++ b/marimo/_runtime/callbacks/cache.py
@@ -0,0 +1,67 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from marimo._messaging.notification import (
+ CacheClearedNotification,
+ CacheInfoNotification,
+)
+from marimo._messaging.notification_utils import broadcast_notification
+from marimo._runtime.commands import ClearCacheCommand, GetCacheInfoCommand
+from marimo._runtime.context import get_context
+
+if TYPE_CHECKING:
+ from marimo._runtime.request_router import RequestRouter
+ from marimo._runtime.runtime import Kernel
+
+
+class CacheCallbacks:
+ def __init__(self, kernel: Kernel):
+ self._kernel = kernel
+
+ def register(self, router: RequestRouter) -> None:
+ router.register(ClearCacheCommand, self.clear_cache)
+ router.register(GetCacheInfoCommand, self.get_cache_info)
+
+ async def clear_cache(self, request: ClearCacheCommand) -> None:
+ del request
+ from marimo._save.cache import CacheContext
+ from marimo._save.loaders import BasePersistenceLoader
+
+ ctx = get_context()
+ saved = 0
+ for obj in ctx.globals.values():
+ if isinstance(obj, CacheContext):
+ if isinstance(obj.loader, BasePersistenceLoader):
+ obj.loader.clear()
+
+ broadcast_notification(CacheClearedNotification(bytes_freed=saved))
+
+ async def get_cache_info(self, request: GetCacheInfoCommand) -> None:
+ del request
+ from marimo._save.cache import CacheContext
+
+ ctx = get_context()
+ total_hits = 0
+ total_misses = 0
+ total_time = 0
+ disk_to_free = -1 # TODO: sum up disk usage
+ disk_total = -1
+
+ for obj in ctx.globals.values():
+ if isinstance(obj, CacheContext):
+ hits, misses, _, _, time = obj.cache_info()
+ total_hits += hits
+ total_misses += misses
+ total_time += time
+ # d2f, dt = obj.loader.disk_usage()
+ broadcast_notification(
+ CacheInfoNotification(
+ hits=total_hits,
+ misses=total_misses,
+ time=total_time,
+ disk_to_free=disk_to_free,
+ disk_total=disk_total,
+ ),
+ )
diff --git a/marimo/_runtime/callbacks/datasets.py b/marimo/_runtime/callbacks/datasets.py
new file mode 100644
index 0000000000..57e7c46a7c
--- /dev/null
+++ b/marimo/_runtime/callbacks/datasets.py
@@ -0,0 +1,348 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, cast
+
+from marimo import _loggers
+from marimo._data.preview_column import (
+ get_column_preview_for_dataframe,
+ get_column_preview_for_duckdb,
+)
+from marimo._messaging.notification import (
+ DataColumnPreviewNotification,
+ DataSourceConnectionsNotification,
+ SQLDatabaseMetadata,
+ SQLMetadata,
+ SQLSchemaListPreviewNotification,
+ SQLTableListPreviewNotification,
+ SQLTablePreviewNotification,
+)
+from marimo._messaging.notification_utils import broadcast_notification
+from marimo._runtime.commands import (
+ ListDataSourceConnectionCommand,
+ ListSQLSchemasCommand,
+ ListSQLTablesCommand,
+ PreviewDatasetColumnCommand,
+ PreviewSQLTableCommand,
+)
+from marimo._sql.engines.types import EngineCatalog
+from marimo._sql.get_engines import engine_to_data_source_connection
+from marimo._tracer import kernel_tracer
+from marimo._types.ids import VariableName
+from marimo._utils.assert_never import assert_never
+
+if TYPE_CHECKING:
+ from marimo._runtime.request_router import RequestRouter
+ from marimo._runtime.runtime import Kernel
+
+LOGGER = _loggers.marimo_logger()
+
+
+class DatasetCallbacks:
+ def __init__(self, kernel: Kernel):
+ self._kernel = kernel
+
+ def register(self, router: RequestRouter) -> None:
+ router.register(
+ PreviewDatasetColumnCommand, self.preview_dataset_column
+ )
+ router.register(PreviewSQLTableCommand, self.preview_sql_table)
+ router.register(ListSQLTablesCommand, self.preview_sql_table_list)
+ router.register(ListSQLSchemasCommand, self.preview_sql_schema_list)
+ router.register(
+ ListDataSourceConnectionCommand,
+ self.preview_datasource_connection,
+ )
+
+ def get_engine_catalog(
+ self, variable_name: str
+ ) -> tuple[EngineCatalog[Any] | None, str | None]:
+ """Get engines that support catalog operations.
+ Returns an error if the connection does not support catalog operations."""
+ variable_name = cast(VariableName, variable_name)
+ connection, error = self._kernel.get_sql_connection(variable_name)
+ if error is not None or connection is None:
+ return None, error
+
+ if isinstance(connection, EngineCatalog):
+ return connection, None
+ else:
+ return None, "Connection does not support catalog operations"
+
+ @kernel_tracer.start_as_current_span("preview_dataset_column")
+ async def preview_dataset_column(
+ self, request: PreviewDatasetColumnCommand
+ ) -> None:
+ """Preview a column of a dataset.
+
+ The dataset is loaded, and the column is displayed in the frontend.
+
+ Args:
+ request (PreviewDatasetColumnRequest): The preview request containing:
+ - table_name: Name of the table
+ - column_name: Name of the column
+ - source_type: Type of data source ("duckdb" or "local")
+ """
+ table_name = request.table_name
+ column_name = request.column_name
+ source_type = request.source_type
+
+ try:
+ if source_type == "duckdb":
+ column_preview = get_column_preview_for_duckdb(
+ fully_qualified_table_name=request.fully_qualified_table_name
+ or table_name,
+ column_name=column_name,
+ )
+ elif source_type == "local":
+ dataset = self._kernel.globals[table_name]
+ column_preview = get_column_preview_for_dataframe(
+ dataset, request
+ )
+ elif source_type == "connection":
+ broadcast_notification(
+ DataColumnPreviewNotification(
+ error="Column preview for connection data sources is not supported",
+ column_name=column_name,
+ table_name=table_name,
+ ),
+ )
+ return
+ elif source_type == "catalog":
+ broadcast_notification(
+ DataColumnPreviewNotification(
+ error="Column preview for catalog data sources is not supported",
+ column_name=column_name,
+ table_name=table_name,
+ ),
+ )
+ return
+ else:
+ assert_never(source_type)
+
+ if column_preview is None:
+ broadcast_notification(
+ DataColumnPreviewNotification(
+ error=f"Column {column_name} not found",
+ column_name=column_name,
+ table_name=table_name,
+ ),
+ )
+ else:
+ broadcast_notification(column_preview)
+ except Exception as e:
+ LOGGER.warning(
+ "Failed to get preview for column %s in table %s",
+ column_name,
+ table_name,
+ exc_info=e,
+ )
+ broadcast_notification(
+ DataColumnPreviewNotification(
+ error=str(e),
+ column_name=column_name,
+ table_name=table_name,
+ ),
+ )
+ return
+
+ @kernel_tracer.start_as_current_span("preview_sql_table")
+ async def preview_sql_table(self, request: PreviewSQLTableCommand) -> None:
+ """Get table details for an SQL table.
+
+ Args:
+ request (PreviewSQLTableRequest): The request containing:
+ - engine: Name of the SQL engine / connection
+ - database: Name of the database
+ - schema: Name of the schema
+ - table_name: Name of the table
+ """
+ variable_name = cast(VariableName, request.engine)
+ database_name = request.database
+ schema_name = request.schema
+ table_name = request.table_name
+ sql_metadata = SQLMetadata(
+ connection=variable_name,
+ database=database_name,
+ schema=schema_name,
+ )
+
+ engine, error = self.get_engine_catalog(variable_name)
+ if error is not None or engine is None:
+ broadcast_notification(
+ SQLTablePreviewNotification(
+ request_id=request.request_id,
+ table=None,
+ error=error,
+ metadata=sql_metadata,
+ ),
+ )
+ return
+
+ try:
+ table = engine.get_table_details(
+ table_name=table_name,
+ schema_name=schema_name,
+ database_name=database_name,
+ )
+
+ broadcast_notification(
+ SQLTablePreviewNotification(
+ request_id=request.request_id,
+ table=table,
+ metadata=sql_metadata,
+ ),
+ )
+ except Exception as e:
+ LOGGER.exception(
+ "Failed to get preview for table %s in schema %s",
+ table_name,
+ schema_name,
+ )
+ broadcast_notification(
+ SQLTablePreviewNotification(
+ request_id=request.request_id,
+ table=None,
+ error="Failed to get table details: " + str(e),
+ metadata=sql_metadata,
+ ),
+ )
+
+ @kernel_tracer.start_as_current_span("preview_sql_table_list")
+ async def preview_sql_table_list(
+ self, request: ListSQLTablesCommand
+ ) -> None:
+ """Get a list of tables from an SQL schema
+
+ Args:
+ request (ListSQLTablesRequest): The request containing:
+ - engine: Name of the SQL engine / connection
+ - database: Name of the database
+ - schema: Name of the schema
+ """
+ variable_name = cast(VariableName, request.engine)
+ database_name = request.database
+ schema_name = request.schema
+ sql_metadata = SQLMetadata(
+ connection=variable_name,
+ database=database_name,
+ schema=schema_name,
+ )
+
+ engine, error = self.get_engine_catalog(variable_name)
+ if error is not None or engine is None:
+ broadcast_notification(
+ SQLTableListPreviewNotification(
+ request_id=request.request_id,
+ tables=[],
+ error=error,
+ metadata=sql_metadata,
+ ),
+ )
+ return
+
+ try:
+ table_list = engine.get_tables_in_schema(
+ schema=schema_name,
+ database=database_name,
+ include_table_details=False,
+ )
+ broadcast_notification(
+ SQLTableListPreviewNotification(
+ request_id=request.request_id,
+ tables=table_list,
+ metadata=sql_metadata,
+ ),
+ )
+ except Exception as e:
+ LOGGER.exception(
+ "Failed to get table list for schema %s", schema_name
+ )
+ broadcast_notification(
+ SQLTableListPreviewNotification(
+ request_id=request.request_id,
+ tables=[],
+ error="Failed to get table list: " + str(e),
+ metadata=sql_metadata,
+ ),
+ )
+
+ @kernel_tracer.start_as_current_span("preview_sql_schema_list")
+ async def preview_sql_schema_list(
+ self, request: ListSQLSchemasCommand
+ ) -> None:
+ """Get a list of schemas from an SQL database
+
+ Args:
+ request (ListSQLSchemasCommand): The request containing:
+ - engine: Name of the SQL engine / connection
+ - database: Name of the database
+ """
+ variable_name = cast(VariableName, request.engine)
+ database_name = request.database
+ sql_db_metadata = SQLDatabaseMetadata(
+ connection=variable_name,
+ database=database_name,
+ )
+
+ engine, error = self.get_engine_catalog(variable_name)
+ if error is not None or engine is None:
+ broadcast_notification(
+ SQLSchemaListPreviewNotification(
+ request_id=request.request_id,
+ schemas=[],
+ error=error,
+ metadata=sql_db_metadata,
+ ),
+ )
+ return
+
+ try:
+ schema_list = engine.get_schemas(
+ database=database_name,
+ include_tables=False,
+ include_table_details=False,
+ )
+ broadcast_notification(
+ SQLSchemaListPreviewNotification(
+ request_id=request.request_id,
+ schemas=schema_list,
+ metadata=sql_db_metadata,
+ ),
+ )
+ except Exception as e:
+ LOGGER.exception(
+ "Failed to get schema list for database %s", database_name
+ )
+ broadcast_notification(
+ SQLSchemaListPreviewNotification(
+ request_id=request.request_id,
+ schemas=[],
+ error="Failed to get schema list: " + str(e),
+ metadata=sql_db_metadata,
+ ),
+ )
+
+ @kernel_tracer.start_as_current_span("preview_datasource_connection")
+ async def preview_datasource_connection(
+ self, request: ListDataSourceConnectionCommand
+ ) -> None:
+ """Broadcasts a datasource connection for a given engine"""
+ variable_name = cast(VariableName, request.engine)
+ engine, error = self.get_engine_catalog(variable_name)
+ if error is not None or engine is None:
+ LOGGER.error("Failed to get engine %s", variable_name)
+ return
+
+ data_source_connection = engine_to_data_source_connection(
+ variable_name, engine
+ )
+
+ LOGGER.debug(
+ "Broadcasting datasource connection for %s engine", variable_name
+ )
+ broadcast_notification(
+ DataSourceConnectionsNotification(
+ connections=[data_source_connection],
+ ),
+ )
diff --git a/marimo/_runtime/callbacks/external_storage.py b/marimo/_runtime/callbacks/external_storage.py
new file mode 100644
index 0000000000..ba8ebcab44
--- /dev/null
+++ b/marimo/_runtime/callbacks/external_storage.py
@@ -0,0 +1,216 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import asyncio
+import os
+from typing import TYPE_CHECKING, Any
+
+from marimo import _loggers
+from marimo._data._external_storage.models import StorageBackend, StorageEntry
+from marimo._messaging.notification import (
+ StorageDownloadReadyNotification,
+ StorageEntriesNotification,
+)
+from marimo._messaging.notification_utils import broadcast_notification
+from marimo._runtime.commands import (
+ StorageDownloadCommand,
+ StorageListEntriesCommand,
+)
+from marimo._runtime.virtual_file.virtual_file import VirtualFile
+from marimo._tracer import kernel_tracer
+from marimo._types.ids import VariableName
+
+if TYPE_CHECKING:
+ from marimo._runtime.request_router import RequestRouter
+ from marimo._runtime.runtime import Kernel
+
+LOGGER = _loggers.marimo_logger()
+
+
+class ExternalStorageCallbacks:
+ _VFILE_TTL_SECONDS = 60
+ _PREVIEW_MAX_BYTES = 1_000_000 # 1 MB
+
+ def __init__(self, kernel: Kernel):
+ self._kernel = kernel
+
+ def register(self, router: RequestRouter) -> None:
+ router.register(StorageListEntriesCommand, self.list_entries)
+ router.register(StorageDownloadCommand, self.download)
+
+ def _get_storage_backend(
+ self, namespace: str
+ ) -> tuple[StorageBackend[Any] | None, str | None]:
+ """Look up a storage backend by variable name from kernel globals.
+
+ Returns (backend, error). If there is error, backend is None.
+ """
+ from marimo._data._external_storage.get_storage import STORAGE_BACKENDS
+
+ variable_name = VariableName(namespace)
+ if variable_name not in self._kernel.globals:
+ return None, f"Variable '{namespace}' not found"
+
+ var = self._kernel.globals[variable_name]
+
+ for backend in STORAGE_BACKENDS:
+ if backend.is_compatible(var):
+ return backend(var, variable_name), None
+ return None, (
+ f"Variable '{namespace}' is not a compatible "
+ "storage backend (expected obstore or fsspec)"
+ )
+
+ def _schedule_vfile_cleanup(self, vfile: VirtualFile) -> None:
+ """Best-effort cleanup of a virtual file after a TTL."""
+ from marimo._runtime.context import get_context
+
+ try:
+ registry = get_context().virtual_file_registry
+ loop = asyncio.get_running_loop()
+ loop.call_later(self._VFILE_TTL_SECONDS, registry.remove, vfile)
+ except Exception:
+ LOGGER.debug(
+ "Could not schedule virtual file cleanup for %s",
+ vfile.filename,
+ )
+
+ @kernel_tracer.start_as_current_span("storage_list_entries")
+ async def list_entries(self, request: StorageListEntriesCommand) -> None:
+ """List storage entries at a given prefix."""
+ backend, error = self._get_storage_backend(request.namespace)
+ if error is not None or backend is None:
+ broadcast_notification(
+ StorageEntriesNotification(
+ request_id=request.request_id,
+ entries=[],
+ namespace=request.namespace,
+ prefix=request.prefix,
+ error=error,
+ ),
+ )
+ return
+
+ # list_entries is synchronous, so we wrap it in asyncio.to_thread
+ def list_entries() -> list[StorageEntry]:
+ return backend.list_entries(
+ prefix=request.prefix, limit=request.limit
+ )
+
+ try:
+ entries = await asyncio.to_thread(list_entries)
+ broadcast_notification(
+ StorageEntriesNotification(
+ request_id=request.request_id,
+ entries=entries,
+ namespace=request.namespace,
+ prefix=request.prefix,
+ ),
+ )
+ except Exception as e:
+ LOGGER.exception(
+ "Failed to list entries for %s at prefix %s",
+ request.namespace,
+ request.prefix,
+ )
+ broadcast_notification(
+ StorageEntriesNotification(
+ request_id=request.request_id,
+ entries=[],
+ namespace=request.namespace,
+ prefix=request.prefix,
+ error=f"Failed to list entries: {e}",
+ ),
+ )
+
+ @kernel_tracer.start_as_current_span("storage_download")
+ async def download(self, request: StorageDownloadCommand) -> None:
+ """
+ Download a storage entry, preferring a signed URL.
+ If preview is true, downloads the first 1MB of the file and returns a same-origin virtual file URL.
+ """
+ backend, error = self._get_storage_backend(request.namespace)
+ if error is not None or backend is None:
+ broadcast_notification(
+ StorageDownloadReadyNotification(
+ request_id=request.request_id,
+ url=None,
+ filename=None,
+ error=error,
+ ),
+ )
+ return
+
+ filename = request.path.rsplit("/", 1)[-1] or "download"
+
+ try:
+ if request.preview:
+ await self._download_preview(backend, request, filename)
+ else:
+ await self._download_full(backend, request, filename)
+ except Exception as e:
+ LOGGER.exception(
+ "Failed to download %s from %s",
+ request.path,
+ request.namespace,
+ )
+ broadcast_notification(
+ StorageDownloadReadyNotification(
+ request_id=request.request_id,
+ url=None,
+ filename=None,
+ error=f"Failed to download: {e}",
+ ),
+ )
+
+ async def _download_full(
+ self,
+ backend: StorageBackend[Any],
+ request: StorageDownloadCommand,
+ filename: str,
+ ) -> None:
+ signed_url = await backend.sign_download_url(request.path)
+ if signed_url is not None:
+ broadcast_notification(
+ StorageDownloadReadyNotification(
+ request_id=request.request_id,
+ url=signed_url,
+ filename=filename,
+ ),
+ )
+ return
+
+ # Signing not supported; fall back to virtual file with TTL
+ result = await backend.download_file(request.path)
+ vfile = VirtualFile.create_and_register(result.file_bytes, result.ext)
+ self._schedule_vfile_cleanup(vfile)
+
+ broadcast_notification(
+ StorageDownloadReadyNotification(
+ request_id=request.request_id,
+ url=vfile.url,
+ filename=result.filename,
+ ),
+ )
+
+ async def _download_preview(
+ self,
+ backend: StorageBackend[Any],
+ request: StorageDownloadCommand,
+ filename: str,
+ ) -> None:
+ """Read partial content and serve via a virtual file with TTL. This is useful to bypass CORS."""
+ data = await backend.read_range(
+ request.path, offset=0, length=self._PREVIEW_MAX_BYTES
+ )
+ _, ext = os.path.splitext(filename)
+ vfile = VirtualFile.create_and_register(data, ext.lstrip(".") or "txt")
+ self._schedule_vfile_cleanup(vfile)
+
+ broadcast_notification(
+ StorageDownloadReadyNotification(
+ request_id=request.request_id,
+ url=vfile.url,
+ filename=filename,
+ ),
+ )
diff --git a/marimo/_runtime/callbacks/packages.py b/marimo/_runtime/callbacks/packages.py
new file mode 100644
index 0000000000..d5f27afe9f
--- /dev/null
+++ b/marimo/_runtime/callbacks/packages.py
@@ -0,0 +1,344 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from marimo import _loggers
+from marimo._config.settings import GLOBAL_SETTINGS
+from marimo._dependencies.errors import ManyModulesNotFoundError
+from marimo._messaging.context import is_code_mode_request
+from marimo._messaging.notification import (
+ CompletedRunNotification,
+ InstallingPackageAlertNotification,
+ MissingPackageAlertNotification,
+ PackageStatusType,
+)
+from marimo._messaging.notification_utils import broadcast_notification
+from marimo._runtime.commands import InstallPackagesCommand
+from marimo._runtime.packages.import_error_extractors import (
+ extract_missing_module_from_cause_chain,
+ try_extract_packages_from_import_error_message,
+)
+from marimo._runtime.packages.package_manager import (
+ LogCallback,
+ PackageManager,
+)
+from marimo._runtime.packages.package_managers import create_package_manager
+from marimo._runtime.packages.utils import (
+ PackageRequirement,
+ is_python_isolated,
+)
+from marimo._runtime.runner import hook_context
+
+if TYPE_CHECKING:
+ from marimo._runtime.request_router import RequestRouter
+ from marimo._runtime.runtime import Kernel
+
+LOGGER = _loggers.marimo_logger()
+
+
+class PackagesCallbacks:
+ def __init__(self, kernel: Kernel):
+ self._kernel = kernel
+ self.package_manager: PackageManager | None = None
+
+ def register(self, router: RequestRouter) -> None:
+ router.register(InstallPackagesCommand, self._handle_install)
+
+ async def _handle_install(self, request: InstallPackagesCommand) -> None:
+ await self.install_missing_packages(request)
+ broadcast_notification(CompletedRunNotification())
+
+ def update_package_manager(self, package_manager: str) -> None:
+ if (
+ self.package_manager is None
+ or package_manager != self.package_manager.name
+ ):
+ self.package_manager = create_package_manager(package_manager)
+
+ # All marimo notebooks depend on the marimo package; if the
+ # notebook already has marimo as a dependency, or an optional
+ # dependency group with marimo, such as marimo[sql], this is a
+ # NOOP.
+ self._maybe_add_marimo_to_script_metadata()
+
+ def send_missing_packages_alert(self, missing_packages: set[str]) -> None:
+ if self.package_manager is None:
+ return
+
+ packages = sorted(
+ pkg
+ for mod in missing_packages
+ if not self.package_manager.attempted_to_install(
+ pkg := self.package_manager.module_to_package(mod)
+ )
+ )
+ # Deleting a cell can make the set of missing packages smaller
+ broadcast_notification(
+ MissingPackageAlertNotification(
+ packages=packages,
+ isolated=is_python_isolated(),
+ ),
+ )
+
+ def missing_packages_hook(
+ self, ctx: hook_context.OnFinishHookContext
+ ) -> None:
+ module_not_found_errors = [
+ e
+ for e in ctx.exceptions.values()
+ if isinstance(e, (ImportError, ManyModulesNotFoundError))
+ ]
+
+ if len(module_not_found_errors) == 0:
+ return
+
+ if self.package_manager is None:
+ return
+
+ missing_modules: set[str] = set()
+ missing_packages: set[str] = set()
+
+ # Populate missing_modules and missing_packages from the errors
+ for e in module_not_found_errors:
+ if isinstance(e, ManyModulesNotFoundError):
+ # filter out packages that we already attempted to install
+ # to prevent an infinite loop
+ missing_packages.update(
+ {
+ pkg
+ for pkg in e.package_names
+ if not self.package_manager.attempted_to_install(pkg)
+ }
+ )
+ continue
+
+ maybe_missing_module = extract_missing_module_from_cause_chain(e)
+ if maybe_missing_module:
+ missing_modules.add(maybe_missing_module)
+ continue
+
+ maybe_missing_packages = (
+ try_extract_packages_from_import_error_message(str(e))
+ )
+ if maybe_missing_packages:
+ missing_packages.update(
+ {
+ pkg
+ for pkg in maybe_missing_packages
+ if not self.package_manager.attempted_to_install(pkg)
+ }
+ )
+
+ # Grab missing modules from module registry and from module not found errors
+ missing_modules = (
+ self._kernel.module_registry.missing_modules() | missing_modules
+ )
+
+ # Convert modules to packages
+ for mod in missing_modules:
+ pkg = self.package_manager.module_to_package(mod)
+ # filter out packages that we already attempted to install
+ # to prevent an infinite loop
+ if not self.package_manager.attempted_to_install(pkg):
+ missing_packages.add(pkg)
+
+ if not missing_packages:
+ return
+
+ packages = sorted(missing_packages)
+ if self.package_manager.should_auto_install():
+ version = {pkg: "" for pkg in packages}
+ self._kernel.enqueue_control_request(
+ InstallPackagesCommand(
+ manager=self.package_manager.name, versions=version
+ )
+ )
+ else:
+ if is_code_mode_request():
+ return
+
+ broadcast_notification(
+ MissingPackageAlertNotification(
+ packages=packages,
+ isolated=is_python_isolated(),
+ ),
+ )
+
+ async def install_missing_packages(
+ self, request: InstallPackagesCommand
+ ) -> None:
+ """Attempts to install packages for modules that cannot be imported
+
+ Runs cells affected by successful installation.
+ """
+ assert self.package_manager is not None, (
+ "Cannot install packages without a package manager"
+ )
+ if request.manager != self.package_manager.name:
+ # Swap out the package manager
+ self.package_manager = create_package_manager(request.manager)
+
+ if not self.package_manager.is_manager_installed():
+ self.package_manager.alert_not_installed()
+ return
+
+ resolved_packages: dict[str, PackageRequirement] = {}
+ for pkg in request.versions:
+ pkg_req = PackageRequirement.parse(pkg)
+ resolved_packages[pkg_req.name] = pkg_req
+
+ # Append all other missing packages from the notebook; the missing
+ # package request only contains the packages from the cell the user
+ # executed.
+ for module in self._kernel.module_registry.missing_modules():
+ pkg_req = PackageRequirement.parse(
+ self.package_manager.module_to_package(module)
+ )
+ if pkg_req.name not in resolved_packages:
+ resolved_packages[pkg_req.name] = pkg_req
+
+ # Convert back to list of package strings
+ missing_packages = [
+ str(pkg)
+ for pkg in sorted(resolved_packages.values(), key=lambda p: p.name)
+ ]
+
+ # Frontend shows package names, not module names
+ package_statuses: PackageStatusType = {
+ pkg: "queued" for pkg in missing_packages
+ }
+ broadcast_notification(
+ InstallingPackageAlertNotification(
+ packages=package_statuses, source=request.source
+ )
+ )
+
+ def create_log_callback(pkg: str) -> LogCallback:
+ def log_callback(log_line: str) -> None:
+ broadcast_notification(
+ InstallingPackageAlertNotification(
+ packages=package_statuses,
+ logs={pkg: log_line},
+ log_status="append",
+ source=request.source,
+ ),
+ )
+
+ return log_callback
+
+ for pkg in missing_packages:
+ if self.package_manager.attempted_to_install(package=pkg):
+ # Already attempted an installation; it must have failed.
+ # Skip the installation.
+ continue
+ package_statuses[pkg] = "installing"
+ broadcast_notification(
+ InstallingPackageAlertNotification(
+ packages=package_statuses, source=request.source
+ )
+ )
+
+ # Send initial "start" log
+ broadcast_notification(
+ InstallingPackageAlertNotification(
+ packages=package_statuses,
+ logs={pkg: f"Installing {pkg}...\n"},
+ log_status="start",
+ source=request.source,
+ )
+ )
+
+ version = request.versions.get(pkg)
+ if await self.package_manager.install(
+ pkg, version=version, log_callback=create_log_callback(pkg)
+ ):
+ package_statuses[pkg] = "installed"
+ # Send final "done" log
+ broadcast_notification(
+ InstallingPackageAlertNotification(
+ packages=package_statuses,
+ logs={pkg: f"Successfully installed {pkg}\n"},
+ log_status="done",
+ source=request.source,
+ ),
+ )
+ else:
+ package_statuses[pkg] = "failed"
+ mod = self.package_manager.package_to_module(pkg)
+ self._kernel.module_registry.excluded_modules.add(mod)
+ # Send final "done" log with error
+ broadcast_notification(
+ InstallingPackageAlertNotification(
+ packages=package_statuses,
+ logs={pkg: f"Failed to install {pkg}\n"},
+ log_status="done",
+ source=request.source,
+ ),
+ )
+
+ installed_modules = [
+ self.package_manager.package_to_module(pkg)
+ for pkg in package_statuses
+ if package_statuses[pkg] == "installed"
+ ]
+
+ # If a package was not installed at cell registration time, it won't
+ # yet be in the script metadata.
+ if self.should_update_script_metadata():
+ self.update_script_metadata(installed_modules)
+
+ # All cells that depend on successfully installed modules are re-run.
+ #
+ # This consists of cells that either statically reference the installed
+ # module, or that previously failed with a ModuleNotFoundError matching
+ # an installed module.
+ cells_to_run = {
+ cid
+ for module in installed_modules
+ if (cid := self._kernel.module_registry.defining_cell(module))
+ is not None
+ }
+
+ for cid, cell in self._kernel.graph.cells.items():
+ if (
+ isinstance(cell.exception, ModuleNotFoundError)
+ and cell.exception.name in installed_modules
+ ):
+ cells_to_run.add(cid)
+
+ if cells_to_run:
+ await self._kernel._if_autorun_then_run_cells(cells_to_run)
+
+ def _maybe_add_marimo_to_script_metadata(self) -> None:
+ if self.should_update_script_metadata():
+ self.update_script_metadata(["marimo"])
+
+ def should_update_script_metadata(self) -> bool:
+ return (
+ GLOBAL_SETTINGS.MANAGE_SCRIPT_METADATA is True
+ and self._kernel.app_metadata.filename is not None
+ and self.package_manager is not None
+ )
+
+ def update_script_metadata(
+ self, import_namespaces_to_add: list[str]
+ ) -> None:
+ filename = self._kernel.app_metadata.filename
+
+ if not filename or not self.package_manager:
+ return
+
+ try:
+ LOGGER.debug(
+ "Updating script metadata: %s. Adding namespaces: %s.",
+ filename,
+ import_namespaces_to_add,
+ )
+ self.package_manager.update_notebook_script_metadata(
+ filepath=filename,
+ import_namespaces_to_add=import_namespaces_to_add,
+ upgrade=False,
+ )
+ except Exception as e:
+ LOGGER.error("Failed to add script metadata to notebook: %s", e)
diff --git a/marimo/_runtime/callbacks/protocol.py b/marimo/_runtime/callbacks/protocol.py
new file mode 100644
index 0000000000..98b735a551
--- /dev/null
+++ b/marimo/_runtime/callbacks/protocol.py
@@ -0,0 +1,14 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+ from marimo._runtime.request_router import RequestRouter
+
+
+@runtime_checkable
+class KernelCallback(Protocol):
+ """A bundle of kernel command handlers that registers itself with a router."""
+
+ def register(self, router: RequestRouter) -> None: ...
diff --git a/marimo/_runtime/callbacks/secrets.py b/marimo/_runtime/callbacks/secrets.py
new file mode 100644
index 0000000000..2378918159
--- /dev/null
+++ b/marimo/_runtime/callbacks/secrets.py
@@ -0,0 +1,39 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from marimo._messaging.notification import SecretKeysResultNotification
+from marimo._messaging.notification_utils import broadcast_notification
+from marimo._runtime.commands import (
+ ListSecretKeysCommand,
+ RefreshSecretsCommand,
+)
+from marimo._secrets.secrets import get_secret_keys
+
+if TYPE_CHECKING:
+ from marimo._runtime.request_router import RequestRouter
+ from marimo._runtime.runtime import Kernel
+
+
+class SecretsCallbacks:
+ def __init__(self, kernel: Kernel):
+ self._kernel = kernel
+
+ def register(self, router: RequestRouter) -> None:
+ router.register(ListSecretKeysCommand, self.list_secrets)
+ router.register(RefreshSecretsCommand, self.refresh_secrets)
+
+ async def list_secrets(self, request: ListSecretKeysCommand) -> None:
+ secrets = get_secret_keys(
+ self._kernel.user_config, self._kernel._original_environ
+ )
+ broadcast_notification(
+ SecretKeysResultNotification(
+ request_id=request.request_id, secrets=secrets
+ ),
+ )
+
+ async def refresh_secrets(self, request: RefreshSecretsCommand) -> None:
+ del request
+ self._kernel.load_dotenv()
diff --git a/marimo/_runtime/callbacks/sql.py b/marimo/_runtime/callbacks/sql.py
new file mode 100644
index 0000000000..3226032092
--- /dev/null
+++ b/marimo/_runtime/callbacks/sql.py
@@ -0,0 +1,136 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, cast
+
+from marimo import _loggers
+from marimo._messaging.notification import ValidateSQLResultNotification
+from marimo._messaging.notification_utils import broadcast_notification
+from marimo._runtime.commands import ValidateSQLCommand
+from marimo._sql.engines.duckdb import INTERNAL_DUCKDB_ENGINE, DuckDBEngine
+from marimo._sql.engines.types import QueryEngine, SQLConnectionType
+from marimo._sql.parse import SqlCatalogCheckResult, parse_sql
+from marimo._tracer import kernel_tracer
+from marimo._types.ids import VariableName
+
+if TYPE_CHECKING:
+ from marimo._runtime.request_router import RequestRouter
+ from marimo._runtime.runtime import Kernel
+
+LOGGER = _loggers.marimo_logger()
+
+
+class SqlCallbacks:
+ def __init__(self, kernel: Kernel):
+ self._kernel = kernel
+
+ def register(self, router: RequestRouter) -> None:
+ router.register(ValidateSQLCommand, self.validate_sql)
+
+ async def _validate_sql_query(self, request: ValidateSQLCommand) -> None:
+ """Validate an SQL query
+
+ This will validate:
+ - the syntax (parsing)
+ - the catalog (table and column names)
+ """
+ request_id = request.request_id
+
+ if request.only_parse:
+ if request.dialect is None:
+ broadcast_notification(
+ ValidateSQLResultNotification(
+ request_id=request_id,
+ error="Dialect is required when only parsing",
+ ),
+ )
+ return
+
+ # Just parse the query (no DB connection required)
+ parse_result, error = parse_sql(request.query, request.dialect)
+ broadcast_notification(
+ ValidateSQLResultNotification(
+ request_id=request_id,
+ parse_result=parse_result,
+ error=error,
+ ),
+ )
+ return
+
+ # Validate against the database
+ # This can be cheap for in-memory engines (duckdb, sqlite)
+ # But potentially expensive and requires an active connection for remote engines
+ # For failed connections, we should not raise an error
+
+ if request.engine is None:
+ broadcast_notification(
+ ValidateSQLResultNotification(
+ request_id=request_id,
+ error="Engine is required for validating catalog",
+ ),
+ )
+ return
+
+ variable_name = cast(VariableName, request.engine)
+ engine: SQLConnectionType | None = None
+ if variable_name == INTERNAL_DUCKDB_ENGINE:
+ engine = DuckDBEngine(connection=None)
+ error = None
+ else:
+ engine, error = self._kernel.get_sql_connection(variable_name)
+
+ if error is not None or engine is None:
+ broadcast_notification(
+ ValidateSQLResultNotification(
+ request_id=request_id,
+ error="Failed to get engine " + variable_name,
+ ),
+ )
+ return
+
+ # Get the parse error for linting
+ parse_result, parse_error = parse_sql(request.query, engine.dialect)
+ if parse_error is not None:
+ # We don't want to fail the validation if there is a parse error
+ LOGGER.debug("Parse error: %s", parse_error)
+
+ if not isinstance(engine, QueryEngine):
+ broadcast_notification(
+ ValidateSQLResultNotification(
+ request_id=request_id,
+ error=f"Engine {variable_name} does not support catalog validation.",
+ parse_result=parse_result,
+ ),
+ )
+ return
+
+ _, error_message = engine.execute_in_explain_mode( # type: ignore
+ request.query, self._kernel.globals
+ )
+ validate_result = SqlCatalogCheckResult(
+ success=error_message is None,
+ error_message=error_message,
+ )
+ broadcast_notification(
+ ValidateSQLResultNotification(
+ request_id=request_id,
+ validate_result=validate_result,
+ parse_result=parse_result,
+ error=None,
+ ),
+ )
+
+ @kernel_tracer.start_as_current_span("validate_sql")
+ async def validate_sql(self, request: ValidateSQLCommand) -> None:
+ """Validate an SQL query"""
+
+ try:
+ await self._validate_sql_query(request)
+ except Exception as e:
+ LOGGER.exception("Failed to validate SQL query")
+ broadcast_notification(
+ ValidateSQLResultNotification(
+ request_id=request.request_id,
+ error="Failed to validate SQL query: " + str(e),
+ ),
+ )
diff --git a/marimo/_runtime/kernel_request_handlers.py b/marimo/_runtime/kernel_request_handlers.py
new file mode 100644
index 0000000000..d271b75c21
--- /dev/null
+++ b/marimo/_runtime/kernel_request_handlers.py
@@ -0,0 +1,185 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from marimo import _loggers
+from marimo._messaging.context import http_request_context
+from marimo._messaging.notebook.document import (
+ NotebookDocument,
+ notebook_document_context,
+)
+from marimo._messaging.notification import (
+ CompletedRunNotification,
+ FunctionCallResultNotification,
+)
+from marimo._messaging.notification_utils import broadcast_notification
+from marimo._plugins.ui._impl.anywidget.init import WIDGET_COMM_MANAGER
+from marimo._runtime.commands import (
+ CreateNotebookCommand,
+ DebugCellCommand,
+ DeleteCellCommand,
+ ExecuteCellsCommand,
+ ExecuteScratchpadCommand,
+ ExecuteStaleCellsCommand,
+ InvokeFunctionCommand,
+ ModelCommand,
+ RenameNotebookCommand,
+ StopKernelCommand,
+ SyncGraphCommand,
+ UpdateCellConfigCommand,
+ UpdateUIElementCommand,
+ UpdateUserConfigCommand,
+)
+from marimo._types.ids import UIElementId
+
+if TYPE_CHECKING:
+ from marimo._runtime.request_router import RequestRouter
+ from marimo._runtime.runtime import Kernel
+
+LOGGER = _loggers.marimo_logger()
+
+
+class KernelRequestHandlers:
+ """Kernel-owned command handlers.
+
+ These handlers wrap kernel methods with cross-cutting concerns
+ (request context, completion notifications) before delegating.
+ """
+
+ def __init__(self, kernel: Kernel) -> None:
+ self._kernel = kernel
+
+ def register(self, router: RequestRouter) -> None:
+ k = self._kernel
+ router.register(CreateNotebookCommand, self._handle_instantiate)
+ router.register(DeleteCellCommand, k.delete_cell)
+ router.register(ExecuteCellsCommand, self._handle_execute_multiple)
+ router.register(SyncGraphCommand, self._handle_sync_graph)
+ router.register(
+ ExecuteScratchpadCommand, self._handle_execute_scratchpad
+ )
+ router.register(ExecuteStaleCellsCommand, self._handle_execute_stale)
+ router.register(InvokeFunctionCommand, self._handle_function_call)
+ router.register(DebugCellCommand, self._handle_pdb_request)
+ router.register(RenameNotebookCommand, self._handle_rename)
+ router.register(UpdateCellConfigCommand, k.set_cell_config)
+ router.register(
+ UpdateUIElementCommand, self._handle_set_ui_element_value
+ )
+ router.register(ModelCommand, self._handle_receive_model_message)
+ router.register(UpdateUserConfigCommand, self._handle_set_user_config)
+ router.register(StopKernelCommand, self._handle_stop)
+
+ async def _handle_instantiate(
+ self, request: CreateNotebookCommand
+ ) -> None:
+ with http_request_context(request.request):
+ await self._kernel.instantiate(request)
+ broadcast_notification(CompletedRunNotification())
+
+ async def _handle_execute_multiple(
+ self, request: ExecuteCellsCommand
+ ) -> None:
+ with http_request_context(request.request):
+ await self._kernel.run(request.execution_requests)
+ broadcast_notification(CompletedRunNotification())
+
+ async def _handle_sync_graph(self, request: SyncGraphCommand) -> None:
+ with http_request_context(None):
+ await self._kernel.sync_graph(
+ request.cells, request.run_ids, request.delete_ids
+ )
+ broadcast_notification(CompletedRunNotification())
+
+ async def _handle_execute_scratchpad(
+ self, request: ExecuteScratchpadCommand
+ ) -> None:
+ doc = (
+ NotebookDocument(list(request.notebook_cells))
+ if request.notebook_cells is not None
+ else None
+ )
+ try:
+ with (
+ notebook_document_context(doc),
+ http_request_context(request.request),
+ ):
+ await self._kernel.run_scratchpad(request.code)
+ finally:
+ # Always emit completion so a waiting ``ScratchCellListener``
+ # doesn't block forever if ``run_scratchpad`` raises.
+ broadcast_notification(
+ CompletedRunNotification(run_id=request.run_id)
+ )
+
+ async def _handle_execute_stale(
+ self, request: ExecuteStaleCellsCommand
+ ) -> None:
+ with http_request_context(request.request):
+ await self._kernel.run_stale_cells()
+ broadcast_notification(CompletedRunNotification())
+
+ async def _handle_set_ui_element_value(
+ self, request: UpdateUIElementCommand
+ ) -> None:
+ with http_request_context(request.request):
+ await self._kernel.set_ui_element_value(
+ request, notify_frontend=False
+ )
+ broadcast_notification(CompletedRunNotification())
+
+ async def _handle_pdb_request(self, request: DebugCellCommand) -> None:
+ await self._kernel.pdb_request(request.cell_id)
+
+ async def _handle_rename(self, request: RenameNotebookCommand) -> None:
+ await self._kernel.rename_file(request.filename)
+
+ async def _handle_receive_model_message(
+ self, request: ModelCommand
+ ) -> None:
+ ui_element_id, state = WIDGET_COMM_MANAGER.receive_comm_message(
+ request
+ )
+
+ # Directly handle the UI element update instead of
+ # re-enqueuing it as a separate command. Re-enqueuing
+ # caused Model+UI interleaving that the batch merger
+ # couldn't collapse (different types), leading to every
+ # drag tick getting its own full cell re-execution.
+ if ui_element_id and state:
+ await self._kernel.set_ui_element_value(
+ UpdateUIElementCommand.from_ids_and_values(
+ [(UIElementId(ui_element_id), state)]
+ ),
+ notify_frontend=False,
+ )
+ broadcast_notification(CompletedRunNotification())
+ elif self._kernel.state_updates:
+ # Callbacks during message processing (e.g. widget observe
+ # handlers) may have called mo.state setters. Process
+ # those pending state updates now.
+ await self._kernel._run_cells(set())
+ broadcast_notification(CompletedRunNotification())
+
+ async def _handle_function_call(
+ self, request: InvokeFunctionCommand
+ ) -> None:
+ status, ret, _ = await self._kernel.function_call_request(request)
+ LOGGER.debug("Function returned with status %s", status)
+ broadcast_notification(
+ FunctionCallResultNotification(
+ function_call_id=request.function_call_id,
+ return_value=ret,
+ status=status,
+ ),
+ )
+
+ async def _handle_set_user_config(
+ self, request: UpdateUserConfigCommand
+ ) -> None:
+ self._kernel.set_user_config(request)
+
+ async def _handle_stop(self, request: StopKernelCommand) -> None:
+ del request
+ return
diff --git a/marimo/_runtime/request_router.py b/marimo/_runtime/request_router.py
new file mode 100644
index 0000000000..4fbdcc508a
--- /dev/null
+++ b/marimo/_runtime/request_router.py
@@ -0,0 +1,32 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from collections.abc import Awaitable, Callable
+
+ from marimo._runtime.commands import CommandMessage
+
+
+class RequestRouter:
+ """Dispatches kernel command messages to registered async handlers."""
+
+ def __init__(self) -> None:
+ self._handlers: dict[
+ type[CommandMessage],
+ Callable[[Any], Awaitable[None]],
+ ] = {}
+
+ def register(
+ self,
+ request_type: type[CommandMessage],
+ handler: Callable[[Any], Awaitable[None]],
+ ) -> None:
+ self._handlers[request_type] = handler
+
+ async def dispatch(self, request: CommandMessage) -> None:
+ handler = self._handlers.get(type(request))
+ if handler:
+ return await handler(request)
+ raise ValueError(f"Unknown request {request}")
diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.py
index bf0075c600..beff5d1b55 100644
--- a/marimo/_runtime/runtime.py
+++ b/marimo/_runtime/runtime.py
@@ -14,7 +14,6 @@ import threading
import time
import traceback
from copy import copy
-from functools import cached_property
from multiprocessing import connection
from pathlib import Path
from typing import (
@@ -36,20 +35,11 @@ from marimo._config.config import (
MarimoConfig,
OnCellChangeType,
)
-from marimo._config.settings import GLOBAL_SETTINGS
-from marimo._data._external_storage.models import StorageBackend, StorageEntry
-from marimo._data.preview_column import (
- get_column_preview_for_dataframe,
- get_column_preview_for_duckdb,
-)
from marimo._dependencies.dependencies import DependencyManager
-from marimo._dependencies.errors import ManyModulesNotFoundError
from marimo._entrypoints.registry import EntryPointRegistry
from marimo._lint.validate_graph import check_for_errors
from marimo._messaging.cell_output import CellChannel
from marimo._messaging.context import (
- http_request_context,
- is_code_mode_request,
run_id_context,
)
from marimo._messaging.errors import (
@@ -61,33 +51,11 @@ from marimo._messaging.errors import (
UnknownError,
)
from marimo._messaging.notebook.changes import ReorderCells, Transaction
-from marimo._messaging.notebook.document import (
- NotebookDocument,
- notebook_document_context,
-)
from marimo._messaging.notification import (
- CacheClearedNotification,
- CacheInfoNotification,
- CompletedRunNotification,
- DataColumnPreviewNotification,
- DataSourceConnectionsNotification,
- FunctionCallResultNotification,
HumanReadableStatus,
- InstallingPackageAlertNotification,
- MissingPackageAlertNotification,
NotebookDocumentTransactionNotification,
- PackageStatusType,
RemoveUIElementsNotification,
- SecretKeysResultNotification,
- SQLDatabaseMetadata,
- SQLMetadata,
- SQLSchemaListPreviewNotification,
- SQLTableListPreviewNotification,
- SQLTablePreviewNotification,
- StorageDownloadReadyNotification,
- StorageEntriesNotification,
UIElementMessageNotification,
- ValidateSQLResultNotification,
VariableDeclarationNotification,
VariablesNotification,
VariableValue,
@@ -118,42 +86,30 @@ from marimo._messaging.variables import create_variable_value
from marimo._output.rich_help import mddoc
from marimo._plugins.core.web_component import JSONType
from marimo._plugins.ui._core.ui_element import MarimoConvertValueException
-from marimo._plugins.ui._impl.anywidget.init import WIDGET_COMM_MANAGER
from marimo._runtime import dataflow, handlers, marimo_pdb, patches
from marimo._runtime.app_meta import AppMeta
+from marimo._runtime.callbacks import (
+ CacheCallbacks,
+ DatasetCallbacks,
+ ExternalStorageCallbacks,
+ KernelCallback,
+ PackagesCallbacks,
+ SecretsCallbacks,
+ SqlCallbacks,
+)
from marimo._runtime.commands import (
AppMetadata,
BatchableCommand,
- ClearCacheCommand,
CodeCompletionCommand,
CommandMessage,
CreateNotebookCommand,
- DebugCellCommand,
DeleteCellCommand,
ExecuteCellCommand,
- ExecuteCellsCommand,
- ExecuteScratchpadCommand,
ExecuteStaleCellsCommand,
- GetCacheInfoCommand,
- InstallPackagesCommand,
InvokeFunctionCommand,
- ListDataSourceConnectionCommand,
- ListSecretKeysCommand,
- ListSQLSchemasCommand,
- ListSQLTablesCommand,
- ModelCommand,
- PreviewDatasetColumnCommand,
- PreviewSQLTableCommand,
- RefreshSecretsCommand,
- RenameNotebookCommand,
- StopKernelCommand,
- StorageDownloadCommand,
- StorageListEntriesCommand,
- SyncGraphCommand,
UpdateCellConfigCommand,
UpdateUIElementCommand,
UpdateUserConfigCommand,
- ValidateSQLCommand,
)
from marimo._runtime.context import (
ContextNotInitializedError,
@@ -166,20 +122,8 @@ from marimo._runtime.context.kernel_context import (
from marimo._runtime.context.utils import get_mode
from marimo._runtime.control_flow import MarimoInterrupt
from marimo._runtime.input_override import getpass_override
-from marimo._runtime.packages.import_error_extractors import (
- extract_missing_module_from_cause_chain,
- try_extract_packages_from_import_error_message,
-)
+from marimo._runtime.kernel_request_handlers import KernelRequestHandlers
from marimo._runtime.packages.module_registry import ModuleRegistry
-from marimo._runtime.packages.package_manager import (
- LogCallback,
- PackageManager,
-)
-from marimo._runtime.packages.package_managers import create_package_manager
-from marimo._runtime.packages.utils import (
- PackageRequirement,
- is_python_isolated,
-)
from marimo._runtime.params import CLIArgs, QueryParams
from marimo._runtime.parent_poller import (
start_parent_poller,
@@ -187,6 +131,7 @@ from marimo._runtime.parent_poller import (
from marimo._runtime.redirect_streams import redirect_streams
from marimo._runtime.reload.autoreload import ModuleReloader
from marimo._runtime.reload.module_watcher import ModuleWatcher
+from marimo._runtime.request_router import RequestRouter
from marimo._runtime.runner import cell_runner, hook_context
from marimo._runtime.runner.hooks import (
NotebookCellHooks,
@@ -194,29 +139,23 @@ from marimo._runtime.runner.hooks import (
)
from marimo._runtime.scratch import SCRATCH_CELL_ID
from marimo._runtime.state import State
-from marimo._runtime.virtual_file.virtual_file import VirtualFile
from marimo._runtime.win32_interrupt_handler import Win32InterruptHandler
from marimo._secrets.load_dotenv import (
load_dotenv_with_fallback,
)
-from marimo._secrets.secrets import get_secret_keys
from marimo._session.model import SessionMode
from marimo._session.queue import QueueType
-from marimo._sql.engines.duckdb import INTERNAL_DUCKDB_ENGINE, DuckDBEngine
from marimo._sql.engines.types import (
EngineCatalog,
QueryEngine,
SQLConnectionType,
)
from marimo._sql.get_engines import (
- engine_to_data_source_connection,
get_engines_from_variables,
)
-from marimo._sql.parse import SqlCatalogCheckResult, parse_sql
from marimo._tracer import kernel_tracer
from marimo._types.ids import CellId_t, UIElementId, VariableName
from marimo._types.lifespan import Lifespan
-from marimo._utils.assert_never import assert_never
from marimo._utils.lifespans import Lifespans
from marimo._utils.paths import normalize_path
from marimo._utils.platform import is_pyodide
@@ -224,7 +163,7 @@ from marimo._utils.signals import restore_signals
from marimo._utils.typed_connection import TypedConnection
if TYPE_CHECKING:
- from collections.abc import Awaitable, Callable, Iterator, Sequence
+ from collections.abc import Callable, Iterator, Sequence
from types import ModuleType
from marimo._plugins.ui._core.ui_element import UIElement
@@ -540,13 +479,27 @@ class Kernel:
# timestamp, to save the user from having to spam the interrupt button
self.last_interrupt_timestamp: float | None = None
- # Callbacks
+ # Named attributes exist because internal kernel paths (run hooks,
+ # script metadata) and tests reach into specific callbacks directly.
self.secrets_callbacks = SecretsCallbacks(self)
self.datasets_callbacks = DatasetCallbacks(self)
self.packages_callbacks = PackagesCallbacks(self)
self.sql_callbacks = SqlCallbacks(self)
self.cache_callbacks = CacheCallbacks(self)
self.external_storage_callbacks = ExternalStorageCallbacks(self)
+ self._callbacks: list[KernelCallback] = [
+ self.secrets_callbacks,
+ self.datasets_callbacks,
+ self.packages_callbacks,
+ self.sql_callbacks,
+ self.cache_callbacks,
+ self.external_storage_callbacks,
+ ]
+
+ self.router = RequestRouter()
+ KernelRequestHandlers(self).register(self.router)
+ for cb in self._callbacks:
+ cb.register(self.router)
# Apply pythonpath from config at initialization
pythonpath = user_config["runtime"].get("pythonpath")
@@ -2348,201 +2301,16 @@ class Kernel:
"Failed to load dotenv file %s", env, exc_info=e
)
- @cached_property
- def request_handler(self) -> RequestHandler:
- handler = RequestHandler()
-
- async def handle_instantiate(request: CreateNotebookCommand) -> None:
- with http_request_context(request.request):
- await self.instantiate(request)
- broadcast_notification(CompletedRunNotification())
-
- async def handle_execute_multiple(
- request: ExecuteCellsCommand,
- ) -> None:
- with http_request_context(request.request):
- await self.run(request.execution_requests)
- broadcast_notification(CompletedRunNotification())
-
- async def handle_sync_graph(
- request: SyncGraphCommand,
- ) -> None:
- with http_request_context(None):
- await self.sync_graph(
- request.cells, request.run_ids, request.delete_ids
- )
- broadcast_notification(CompletedRunNotification())
-
- async def handle_execute_scratchpad(
- request: ExecuteScratchpadCommand,
- ) -> None:
- doc = (
- NotebookDocument(list(request.notebook_cells))
- if request.notebook_cells is not None
- else None
- )
- try:
- with (
- notebook_document_context(doc),
- http_request_context(request.request),
- ):
- await self.run_scratchpad(request.code)
- finally:
- # Always emit completion so a waiting ``ScratchCellListener``
- # doesn't block forever if ``run_scratchpad`` raises.
- broadcast_notification(
- CompletedRunNotification(run_id=request.run_id)
- )
-
- async def handle_execute_stale(
- request: ExecuteStaleCellsCommand,
- ) -> None:
- with http_request_context(request.request):
- await self.run_stale_cells()
- broadcast_notification(CompletedRunNotification())
-
- async def handle_set_ui_element_value(
- request: UpdateUIElementCommand,
- ) -> None:
- with http_request_context(request.request):
- await self.set_ui_element_value(request, notify_frontend=False)
- broadcast_notification(CompletedRunNotification())
-
- async def handle_pdb_request(request: DebugCellCommand) -> None:
- await self.pdb_request(request.cell_id)
-
- async def handle_rename(request: RenameNotebookCommand) -> None:
- await self.rename_file(request.filename)
-
- async def handle_receive_model_message(
- request: ModelCommand,
- ) -> None:
- ui_element_id, state = WIDGET_COMM_MANAGER.receive_comm_message(
- request
- )
-
- # Directly handle the UI element update instead of
- # re-enqueuing it as a separate command. Re-enqueuing
- # caused Model+UI interleaving that the batch merger
- # couldn't collapse (different types), leading to every
- # drag tick getting its own full cell re-execution.
- if ui_element_id and state:
- await self.set_ui_element_value(
- UpdateUIElementCommand.from_ids_and_values(
- [(UIElementId(ui_element_id), state)]
- ),
- notify_frontend=False,
- )
- broadcast_notification(CompletedRunNotification())
- elif self.state_updates:
- # Callbacks during message processing (e.g. widget observe
- # handlers) may have called mo.state setters. Process
- # those pending state updates now.
- await self._run_cells(set())
- broadcast_notification(CompletedRunNotification())
-
- async def handle_function_call(request: InvokeFunctionCommand) -> None:
- status, ret, _ = await self.function_call_request(request)
- LOGGER.debug("Function returned with status %s", status)
- broadcast_notification(
- FunctionCallResultNotification(
- function_call_id=request.function_call_id,
- return_value=ret,
- status=status,
- ),
- )
-
- async def handle_set_user_config(
- request: UpdateUserConfigCommand,
- ) -> None:
- self.set_user_config(request)
-
- async def handle_install_missing_packages(
- request: InstallPackagesCommand,
- ) -> None:
- await self.packages_callbacks.install_missing_packages(request)
- broadcast_notification(CompletedRunNotification())
-
- async def handle_stop(request: StopKernelCommand) -> None:
- del request
- return
-
- handler.register(CreateNotebookCommand, handle_instantiate)
- handler.register(DeleteCellCommand, self.delete_cell)
- handler.register(ExecuteCellsCommand, handle_execute_multiple)
- handler.register(SyncGraphCommand, handle_sync_graph)
- handler.register(ExecuteScratchpadCommand, handle_execute_scratchpad)
- handler.register(ExecuteStaleCellsCommand, handle_execute_stale)
- handler.register(InvokeFunctionCommand, handle_function_call)
- handler.register(
- InstallPackagesCommand, handle_install_missing_packages
- )
- handler.register(DebugCellCommand, handle_pdb_request)
- handler.register(RenameNotebookCommand, handle_rename)
- handler.register(UpdateCellConfigCommand, self.set_cell_config)
- handler.register(UpdateUIElementCommand, handle_set_ui_element_value)
- handler.register(ModelCommand, handle_receive_model_message)
- handler.register(UpdateUserConfigCommand, handle_set_user_config)
- handler.register(StopKernelCommand, handle_stop)
- # Datasets
- handler.register(
- PreviewDatasetColumnCommand,
- self.datasets_callbacks.preview_dataset_column,
- )
- handler.register(
- PreviewSQLTableCommand, self.datasets_callbacks.preview_sql_table
- )
- handler.register(
- ListSQLTablesCommand,
- self.datasets_callbacks.preview_sql_table_list,
- )
- handler.register(
- ListSQLSchemasCommand,
- self.datasets_callbacks.preview_sql_schema_list,
- )
- handler.register(
- ListDataSourceConnectionCommand,
- self.datasets_callbacks.preview_datasource_connection,
- )
- # SQL
- handler.register(ValidateSQLCommand, self.sql_callbacks.validate_sql)
- # External storage
- handler.register(
- StorageListEntriesCommand,
- self.external_storage_callbacks.list_entries,
- )
- handler.register(
- StorageDownloadCommand, self.external_storage_callbacks.download
- )
- # Secrets
- handler.register(
- ListSecretKeysCommand, self.secrets_callbacks.list_secrets
- )
- handler.register(
- RefreshSecretsCommand, self.secrets_callbacks.refresh_secrets
- )
- # Cache
- handler.register(ClearCacheCommand, self.cache_callbacks.clear_cache)
- handler.register(
- GetCacheInfoCommand, self.cache_callbacks.get_cache_info
- )
-
- return handler
-
async def handle_message(self, request: CommandMessage) -> None:
"""Handle a message from the client.
- The message is dispatched to the appropriate method based on its type.
-
- Coarsely locks globals to avoid race conditions with code completion.
+ Coarsely locks globals to avoid race conditions with code completion;
+ acquiring an RLock costs ~100ns so the overhead is negligible.
"""
- # acquiring and releasing an RLock takes ~100ns; the overhead is
- # negligible because the lock is coarse.
LOGGER.debug("Acquiring globals lock to handle request %s", request)
-
with self.lock_globals():
LOGGER.debug("Handling control request: %s", request)
- await self.request_handler.handle(request)
+ await self.router.dispatch(request)
LOGGER.debug("Handled control request: %s", request)
def get_sql_connection(
@@ -2574,992 +2342,6 @@ class Kernel:
return None, str(e)
-class DatasetCallbacks:
- def __init__(self, kernel: Kernel):
- self._kernel = kernel
-
- def get_engine_catalog(
- self, variable_name: str
- ) -> tuple[EngineCatalog[Any] | None, str | None]:
- """Get engines that support catalog operations.
- Returns an error if the connection does not support catalog operations."""
- variable_name = cast(VariableName, variable_name)
- connection, error = self._kernel.get_sql_connection(variable_name)
- if error is not None or connection is None:
- return None, error
-
- if isinstance(connection, EngineCatalog):
- return connection, None
- else:
- return None, "Connection does not support catalog operations"
-
- @kernel_tracer.start_as_current_span("preview_dataset_column")
- async def preview_dataset_column(
- self, request: PreviewDatasetColumnCommand
- ) -> None:
- """Preview a column of a dataset.
-
- The dataset is loaded, and the column is displayed in the frontend.
-
- Args:
- request (PreviewDatasetColumnRequest): The preview request containing:
- - table_name: Name of the table
- - column_name: Name of the column
- - source_type: Type of data source ("duckdb" or "local")
- """
- table_name = request.table_name
- column_name = request.column_name
- source_type = request.source_type
-
- try:
- if source_type == "duckdb":
- column_preview = get_column_preview_for_duckdb(
- fully_qualified_table_name=request.fully_qualified_table_name
- or table_name,
- column_name=column_name,
- )
- elif source_type == "local":
- dataset = self._kernel.globals[table_name]
- column_preview = get_column_preview_for_dataframe(
- dataset, request
- )
- elif source_type == "connection":
- broadcast_notification(
- DataColumnPreviewNotification(
- error="Column preview for connection data sources is not supported",
- column_name=column_name,
- table_name=table_name,
- ),
- )
- return
- elif source_type == "catalog":
- broadcast_notification(
- DataColumnPreviewNotification(
- error="Column preview for catalog data sources is not supported",
- column_name=column_name,
- table_name=table_name,
- ),
- )
- return
- else:
- assert_never(source_type)
-
- if column_preview is None:
- broadcast_notification(
- DataColumnPreviewNotification(
- error=f"Column {column_name} not found",
- column_name=column_name,
- table_name=table_name,
- ),
- )
- else:
- broadcast_notification(column_preview)
- except Exception as e:
- LOGGER.warning(
- "Failed to get preview for column %s in table %s",
- column_name,
- table_name,
- exc_info=e,
- )
- broadcast_notification(
- DataColumnPreviewNotification(
- error=str(e),
- column_name=column_name,
- table_name=table_name,
- ),
- )
- return
-
- @kernel_tracer.start_as_current_span("preview_sql_table")
- async def preview_sql_table(self, request: PreviewSQLTableCommand) -> None:
- """Get table details for an SQL table.
-
- Args:
- request (PreviewSQLTableRequest): The request containing:
- - engine: Name of the SQL engine / connection
- - database: Name of the database
- - schema: Name of the schema
- - table_name: Name of the table
- """
- variable_name = cast(VariableName, request.engine)
- database_name = request.database
- schema_name = request.schema
- table_name = request.table_name
- sql_metadata = SQLMetadata(
- connection=variable_name,
- database=database_name,
- schema=schema_name,
- )
-
- engine, error = self.get_engine_catalog(variable_name)
- if error is not None or engine is None:
- broadcast_notification(
- SQLTablePreviewNotification(
- request_id=request.request_id,
- table=None,
- error=error,
- metadata=sql_metadata,
- ),
- )
- return
-
- try:
- table = engine.get_table_details(
- table_name=table_name,
- schema_name=schema_name,
- database_name=database_name,
- )
-
- broadcast_notification(
- SQLTablePreviewNotification(
- request_id=request.request_id,
- table=table,
- metadata=sql_metadata,
- ),
- )
- except Exception as e:
- LOGGER.exception(
- "Failed to get preview for table %s in schema %s",
- table_name,
- schema_name,
- )
- broadcast_notification(
- SQLTablePreviewNotification(
- request_id=request.request_id,
- table=None,
- error="Failed to get table details: " + str(e),
- metadata=sql_metadata,
- ),
- )
-
- @kernel_tracer.start_as_current_span("preview_sql_table_list")
- async def preview_sql_table_list(
- self, request: ListSQLTablesCommand
- ) -> None:
- """Get a list of tables from an SQL schema
-
- Args:
- request (ListSQLTablesRequest): The request containing:
- - engine: Name of the SQL engine / connection
- - database: Name of the database
- - schema: Name of the schema
- """
- variable_name = cast(VariableName, request.engine)
- database_name = request.database
- schema_name = request.schema
- sql_metadata = SQLMetadata(
- connection=variable_name,
- database=database_name,
- schema=schema_name,
- )
-
- engine, error = self.get_engine_catalog(variable_name)
- if error is not None or engine is None:
- broadcast_notification(
- SQLTableListPreviewNotification(
- request_id=request.request_id,
- tables=[],
- error=error,
- metadata=sql_metadata,
- ),
- )
- return
-
- try:
- table_list = engine.get_tables_in_schema(
- schema=schema_name,
- database=database_name,
- include_table_details=False,
- )
- broadcast_notification(
- SQLTableListPreviewNotification(
- request_id=request.request_id,
- tables=table_list,
- metadata=sql_metadata,
- ),
- )
- except Exception as e:
- LOGGER.exception(
- "Failed to get table list for schema %s", schema_name
- )
- broadcast_notification(
- SQLTableListPreviewNotification(
- request_id=request.request_id,
- tables=[],
- error="Failed to get table list: " + str(e),
- metadata=sql_metadata,
- ),
- )
-
- @kernel_tracer.start_as_current_span("preview_sql_schema_list")
- async def preview_sql_schema_list(
- self, request: ListSQLSchemasCommand
- ) -> None:
- """Get a list of schemas from an SQL database
-
- Args:
- request (ListSQLSchemasCommand): The request containing:
- - engine: Name of the SQL engine / connection
- - database: Name of the database
- """
- variable_name = cast(VariableName, request.engine)
- database_name = request.database
- sql_db_metadata = SQLDatabaseMetadata(
- connection=variable_name,
- database=database_name,
- )
-
- engine, error = self.get_engine_catalog(variable_name)
- if error is not None or engine is None:
- broadcast_notification(
- SQLSchemaListPreviewNotification(
- request_id=request.request_id,
- schemas=[],
- error=error,
- metadata=sql_db_metadata,
- ),
- )
- return
-
- try:
- schema_list = engine.get_schemas(
- database=database_name,
- include_tables=False,
- include_table_details=False,
- )
- broadcast_notification(
- SQLSchemaListPreviewNotification(
- request_id=request.request_id,
- schemas=schema_list,
- metadata=sql_db_metadata,
- ),
- )
- except Exception as e:
- LOGGER.exception(
- "Failed to get schema list for database %s", database_name
- )
- broadcast_notification(
- SQLSchemaListPreviewNotification(
- request_id=request.request_id,
- schemas=[],
- error="Failed to get schema list: " + str(e),
- metadata=sql_db_metadata,
- ),
- )
-
- @kernel_tracer.start_as_current_span("preview_datasource_connection")
- async def preview_datasource_connection(
- self, request: ListDataSourceConnectionCommand
- ) -> None:
- """Broadcasts a datasource connection for a given engine"""
- variable_name = cast(VariableName, request.engine)
- engine, error = self.get_engine_catalog(variable_name)
- if error is not None or engine is None:
- LOGGER.error("Failed to get engine %s", variable_name)
- return
-
- data_source_connection = engine_to_data_source_connection(
- variable_name, engine
- )
-
- LOGGER.debug(
- "Broadcasting datasource connection for %s engine", variable_name
- )
- broadcast_notification(
- DataSourceConnectionsNotification(
- connections=[data_source_connection],
- ),
- )
-
-
-class ExternalStorageCallbacks:
- def __init__(self, kernel: Kernel):
- self._kernel = kernel
-
- def _get_storage_backend(
- self, namespace: str
- ) -> tuple[StorageBackend[Any] | None, str | None]:
- """Look up a storage backend by variable name from kernel globals.
-
- Returns (backend, error). If there is error, backend is None.
- """
- from marimo._data._external_storage.get_storage import STORAGE_BACKENDS
-
- variable_name = VariableName(namespace)
- if variable_name not in self._kernel.globals:
- return None, f"Variable '{namespace}' not found"
-
- var = self._kernel.globals[variable_name]
-
- for backend in STORAGE_BACKENDS:
- if backend.is_compatible(var):
- return backend(var, variable_name), None
- return None, (
- f"Variable '{namespace}' is not a compatible "
- "storage backend (expected obstore or fsspec)"
- )
-
- _VFILE_TTL_SECONDS = 60
-
- def _schedule_vfile_cleanup(self, vfile: VirtualFile) -> None:
- """Best-effort cleanup of a virtual file after a TTL."""
- import asyncio
-
- from marimo._runtime.context import get_context
-
- try:
- registry = get_context().virtual_file_registry
- loop = asyncio.get_running_loop()
- loop.call_later(self._VFILE_TTL_SECONDS, registry.remove, vfile)
- except Exception:
- LOGGER.debug(
- "Could not schedule virtual file cleanup for %s",
- vfile.filename,
- )
-
- @kernel_tracer.start_as_current_span("storage_list_entries")
- async def list_entries(self, request: StorageListEntriesCommand) -> None:
- """List storage entries at a given prefix."""
- backend, error = self._get_storage_backend(request.namespace)
- if error is not None or backend is None:
- broadcast_notification(
- StorageEntriesNotification(
- request_id=request.request_id,
- entries=[],
- namespace=request.namespace,
- prefix=request.prefix,
- error=error,
- ),
- )
- return
-
- # list_entries is synchronous, so we wrap it in asyncio.to_thread
- def list_entries() -> list[StorageEntry]:
- return backend.list_entries(
- prefix=request.prefix, limit=request.limit
- )
-
- try:
- entries = await asyncio.to_thread(list_entries)
- broadcast_notification(
- StorageEntriesNotification(
- request_id=request.request_id,
- entries=entries,
- namespace=request.namespace,
- prefix=request.prefix,
- ),
- )
- except Exception as e:
- LOGGER.exception(
- "Failed to list entries for %s at prefix %s",
- request.namespace,
- request.prefix,
- )
- broadcast_notification(
- StorageEntriesNotification(
- request_id=request.request_id,
- entries=[],
- namespace=request.namespace,
- prefix=request.prefix,
- error=f"Failed to list entries: {e}",
- ),
- )
-
- _PREVIEW_MAX_BYTES = 1_000_000 # 1 MB
-
- @kernel_tracer.start_as_current_span("storage_download")
- async def download(self, request: StorageDownloadCommand) -> None:
- """
- Download a storage entry, preferring a signed URL.
- If preview is true, downloads the first 1MB of the file and returns a same-origin virtual file URL.
- """
- backend, error = self._get_storage_backend(request.namespace)
- if error is not None or backend is None:
- broadcast_notification(
- StorageDownloadReadyNotification(
- request_id=request.request_id,
- url=None,
- filename=None,
- error=error,
- ),
- )
- return
-
- filename = request.path.rsplit("/", 1)[-1] or "download"
-
- try:
- if request.preview:
- await self._download_preview(backend, request, filename)
- else:
- await self._download_full(backend, request, filename)
- except Exception as e:
- LOGGER.exception(
- "Failed to download %s from %s",
- request.path,
- request.namespace,
- )
- broadcast_notification(
- StorageDownloadReadyNotification(
- request_id=request.request_id,
- url=None,
- filename=None,
- error=f"Failed to download: {e}",
- ),
- )
-
- async def _download_full(
- self,
- backend: StorageBackend[Any],
- request: StorageDownloadCommand,
- filename: str,
- ) -> None:
- signed_url = await backend.sign_download_url(request.path)
- if signed_url is not None:
- broadcast_notification(
- StorageDownloadReadyNotification(
- request_id=request.request_id,
- url=signed_url,
- filename=filename,
- ),
- )
- return
-
- # Signing not supported; fall back to virtual file with TTL
- result = await backend.download_file(request.path)
- vfile = VirtualFile.create_and_register(result.file_bytes, result.ext)
- self._schedule_vfile_cleanup(vfile)
-
- broadcast_notification(
- StorageDownloadReadyNotification(
- request_id=request.request_id,
- url=vfile.url,
- filename=result.filename,
- ),
- )
-
- async def _download_preview(
- self,
- backend: StorageBackend[Any],
- request: StorageDownloadCommand,
- filename: str,
- ) -> None:
- """Read partial content and serve via a virtual file with TTL. This is useful to bypass CORS."""
- data = await backend.read_range(
- request.path, offset=0, length=self._PREVIEW_MAX_BYTES
- )
- _, ext = os.path.splitext(filename)
- vfile = VirtualFile.create_and_register(data, ext.lstrip(".") or "txt")
- self._schedule_vfile_cleanup(vfile)
-
- broadcast_notification(
- StorageDownloadReadyNotification(
- request_id=request.request_id,
- url=vfile.url,
- filename=filename,
- ),
- )
-
-
-class SqlCallbacks:
- def __init__(self, kernel: Kernel):
- self._kernel = kernel
-
- async def _validate_sql_query(self, request: ValidateSQLCommand) -> None:
- """Validate an SQL query
-
- This will validate:
- - the syntax (parsing)
- - the catalog (table and column names)
- """
- request_id = request.request_id
-
- if request.only_parse:
- if request.dialect is None:
- broadcast_notification(
- ValidateSQLResultNotification(
- request_id=request_id,
- error="Dialect is required when only parsing",
- ),
- )
- return
-
- # Just parse the query (no DB connection required)
- parse_result, error = parse_sql(request.query, request.dialect)
- broadcast_notification(
- ValidateSQLResultNotification(
- request_id=request_id,
- parse_result=parse_result,
- error=error,
- ),
- )
- return
-
- # Validate against the database
- # This can be cheap for in-memory engines (duckdb, sqlite)
- # But potentially expensive and requires an active connection for remote engines
- # For failed connections, we should not raise an error
-
- if request.engine is None:
- broadcast_notification(
- ValidateSQLResultNotification(
- request_id=request_id,
- error="Engine is required for validating catalog",
- ),
- )
- return
-
- variable_name = cast(VariableName, request.engine)
- engine: SQLConnectionType | None = None
- if variable_name == INTERNAL_DUCKDB_ENGINE:
- engine = DuckDBEngine(connection=None)
- error = None
- else:
- engine, error = self._kernel.get_sql_connection(variable_name)
-
- if error is not None or engine is None:
- broadcast_notification(
- ValidateSQLResultNotification(
- request_id=request_id,
- error="Failed to get engine " + variable_name,
- ),
- )
- return
-
- # Get the parse error for linting
- parse_result, parse_error = parse_sql(request.query, engine.dialect)
- if parse_error is not None:
- # We don't want to fail the validation if there is a parse error
- LOGGER.debug("Parse error: %s", parse_error)
-
- if not isinstance(engine, QueryEngine):
- broadcast_notification(
- ValidateSQLResultNotification(
- request_id=request_id,
- error=f"Engine {variable_name} does not support catalog validation.",
- parse_result=parse_result,
- ),
- )
- return
-
- _, error_message = engine.execute_in_explain_mode( # type: ignore
- request.query, self._kernel.globals
- )
- validate_result = SqlCatalogCheckResult(
- success=error_message is None,
- error_message=error_message,
- )
- broadcast_notification(
- ValidateSQLResultNotification(
- request_id=request_id,
- validate_result=validate_result,
- parse_result=parse_result,
- error=None,
- ),
- )
-
- @kernel_tracer.start_as_current_span("validate_sql")
- async def validate_sql(self, request: ValidateSQLCommand) -> None:
- """Validate an SQL query"""
-
- try:
- await self._validate_sql_query(request)
- except Exception as e:
- LOGGER.exception("Failed to validate SQL query")
- broadcast_notification(
- ValidateSQLResultNotification(
- request_id=request.request_id,
- error="Failed to validate SQL query: " + str(e),
- ),
- )
-
-
-class SecretsCallbacks:
- def __init__(self, kernel: Kernel):
- self._kernel = kernel
-
- async def list_secrets(self, request: ListSecretKeysCommand) -> None:
- secrets = get_secret_keys(
- self._kernel.user_config, self._kernel._original_environ
- )
- broadcast_notification(
- SecretKeysResultNotification(
- request_id=request.request_id, secrets=secrets
- ),
- )
-
- async def refresh_secrets(self, request: RefreshSecretsCommand) -> None:
- del request
- self._kernel.load_dotenv()
-
-
-class PackagesCallbacks:
- def __init__(self, kernel: Kernel):
- self._kernel = kernel
- self.package_manager: PackageManager | None = None
-
- def update_package_manager(self, package_manager: str) -> None:
- if (
- self.package_manager is None
- or package_manager != self.package_manager.name
- ):
- self.package_manager = create_package_manager(package_manager)
-
- # All marimo notebooks depend on the marimo package; if the
- # notebook already has marimo as a dependency, or an optional
- # dependency group with marimo, such as marimo[sql], this is a
- # NOOP.
- self._maybe_add_marimo_to_script_metadata()
-
- def send_missing_packages_alert(self, missing_packages: set[str]) -> None:
- if self.package_manager is None:
- return
-
- packages = sorted(
- pkg
- for mod in missing_packages
- if not self.package_manager.attempted_to_install(
- pkg := self.package_manager.module_to_package(mod)
- )
- )
- # Deleting a cell can make the set of missing packages smaller
- broadcast_notification(
- MissingPackageAlertNotification(
- packages=packages,
- isolated=is_python_isolated(),
- ),
- )
-
- def missing_packages_hook(
- self, ctx: hook_context.OnFinishHookContext
- ) -> None:
- module_not_found_errors = [
- e
- for e in ctx.exceptions.values()
- if isinstance(e, (ImportError, ManyModulesNotFoundError))
- ]
-
- if len(module_not_found_errors) == 0:
- return
-
- if self.package_manager is None:
- return
-
- missing_modules: set[str] = set()
- missing_packages: set[str] = set()
-
- # Populate missing_modules and missing_packages from the errors
- for e in module_not_found_errors:
- if isinstance(e, ManyModulesNotFoundError):
- # filter out packages that we already attempted to install
- # to prevent an infinite loop
- missing_packages.update(
- {
- pkg
- for pkg in e.package_names
- if not self.package_manager.attempted_to_install(pkg)
- }
- )
- continue
-
- maybe_missing_module = extract_missing_module_from_cause_chain(e)
- if maybe_missing_module:
- missing_modules.add(maybe_missing_module)
- continue
-
- maybe_missing_packages = (
- try_extract_packages_from_import_error_message(str(e))
- )
- if maybe_missing_packages:
- missing_packages.update(
- {
- pkg
- for pkg in maybe_missing_packages
- if not self.package_manager.attempted_to_install(pkg)
- }
- )
-
- # Grab missing modules from module registry and from module not found errors
- missing_modules = (
- self._kernel.module_registry.missing_modules() | missing_modules
- )
-
- # Convert modules to packages
- for mod in missing_modules:
- pkg = self.package_manager.module_to_package(mod)
- # filter out packages that we already attempted to install
- # to prevent an infinite loop
- if not self.package_manager.attempted_to_install(pkg):
- missing_packages.add(pkg)
-
- if not missing_packages:
- return
-
- packages = sorted(missing_packages)
- if self.package_manager.should_auto_install():
- version = {pkg: "" for pkg in packages}
- self._kernel.enqueue_control_request(
- InstallPackagesCommand(
- manager=self.package_manager.name, versions=version
- )
- )
- else:
- if is_code_mode_request():
- return
-
- broadcast_notification(
- MissingPackageAlertNotification(
- packages=packages,
- isolated=is_python_isolated(),
- ),
- )
-
- async def install_missing_packages(
- self, request: InstallPackagesCommand
- ) -> None:
- """Attempts to install packages for modules that cannot be imported
-
- Runs cells affected by successful installation.
- """
- assert self.package_manager is not None, (
- "Cannot install packages without a package manager"
- )
- if request.manager != self.package_manager.name:
- # Swap out the package manager
- self.package_manager = create_package_manager(request.manager)
-
- if not self.package_manager.is_manager_installed():
- self.package_manager.alert_not_installed()
- return
-
- resolved_packages: dict[str, PackageRequirement] = {}
- for pkg in request.versions:
- pkg_req = PackageRequirement.parse(pkg)
- resolved_packages[pkg_req.name] = pkg_req
-
- # Append all other missing packages from the notebook; the missing
- # package request only contains the packages from the cell the user
- # executed.
- for module in self._kernel.module_registry.missing_modules():
- pkg_req = PackageRequirement.parse(
- self.package_manager.module_to_package(module)
- )
- if pkg_req.name not in resolved_packages:
- resolved_packages[pkg_req.name] = pkg_req
-
- # Convert back to list of package strings
- missing_packages = [
- str(pkg)
- for pkg in sorted(resolved_packages.values(), key=lambda p: p.name)
- ]
-
- # Frontend shows package names, not module names
- package_statuses: PackageStatusType = {
- pkg: "queued" for pkg in missing_packages
- }
- broadcast_notification(
- InstallingPackageAlertNotification(
- packages=package_statuses, source=request.source
- )
- )
-
- def create_log_callback(pkg: str) -> LogCallback:
- def log_callback(log_line: str) -> None:
- broadcast_notification(
- InstallingPackageAlertNotification(
- packages=package_statuses,
- logs={pkg: log_line},
- log_status="append",
- source=request.source,
- ),
- )
-
- return log_callback
-
- for pkg in missing_packages:
- if self.package_manager.attempted_to_install(package=pkg):
- # Already attempted an installation; it must have failed.
- # Skip the installation.
- continue
- package_statuses[pkg] = "installing"
- broadcast_notification(
- InstallingPackageAlertNotification(
- packages=package_statuses, source=request.source
- )
- )
-
- # Send initial "start" log
- broadcast_notification(
- InstallingPackageAlertNotification(
- packages=package_statuses,
- logs={pkg: f"Installing {pkg}...\n"},
- log_status="start",
- source=request.source,
- )
- )
-
- version = request.versions.get(pkg)
- if await self.package_manager.install(
- pkg, version=version, log_callback=create_log_callback(pkg)
- ):
- package_statuses[pkg] = "installed"
- # Send final "done" log
- broadcast_notification(
- InstallingPackageAlertNotification(
- packages=package_statuses,
- logs={pkg: f"Successfully installed {pkg}\n"},
- log_status="done",
- source=request.source,
- ),
- )
- else:
- package_statuses[pkg] = "failed"
- mod = self.package_manager.package_to_module(pkg)
- self._kernel.module_registry.excluded_modules.add(mod)
- # Send final "done" log with error
- broadcast_notification(
- InstallingPackageAlertNotification(
- packages=package_statuses,
- logs={pkg: f"Failed to install {pkg}\n"},
- log_status="done",
- source=request.source,
- ),
- )
-
- installed_modules = [
- self.package_manager.package_to_module(pkg)
- for pkg in package_statuses
- if package_statuses[pkg] == "installed"
- ]
-
- # If a package was not installed at cell registration time, it won't
- # yet be in the script metadata.
- if self.should_update_script_metadata():
- self.update_script_metadata(installed_modules)
-
- # All cells that depend on successfully installed modules are re-run.
- #
- # This consists of cells that either statically reference the installed
- # module, or that previously failed with a ModuleNotFoundError matching
- # an installed module.
- cells_to_run = {
- cid
- for module in installed_modules
- if (cid := self._kernel.module_registry.defining_cell(module))
- is not None
- }
-
- for cid, cell in self._kernel.graph.cells.items():
- if (
- isinstance(cell.exception, ModuleNotFoundError)
- and cell.exception.name in installed_modules
- ):
- cells_to_run.add(cid)
-
- if cells_to_run:
- await self._kernel._if_autorun_then_run_cells(cells_to_run)
-
- def _maybe_add_marimo_to_script_metadata(self) -> None:
- if self.should_update_script_metadata():
- self.update_script_metadata(["marimo"])
-
- def should_update_script_metadata(self) -> bool:
- return (
- GLOBAL_SETTINGS.MANAGE_SCRIPT_METADATA is True
- and self._kernel.app_metadata.filename is not None
- and self.package_manager is not None
- )
-
- def update_script_metadata(
- self, import_namespaces_to_add: list[str]
- ) -> None:
- filename = self._kernel.app_metadata.filename
-
- if not filename or not self.package_manager:
- return
-
- try:
- LOGGER.debug(
- "Updating script metadata: %s. Adding namespaces: %s.",
- filename,
- import_namespaces_to_add,
- )
- self.package_manager.update_notebook_script_metadata(
- filepath=filename,
- import_namespaces_to_add=import_namespaces_to_add,
- upgrade=False,
- )
- except Exception as e:
- LOGGER.error("Failed to add script metadata to notebook: %s", e)
-
-
-class CacheCallbacks:
- def __init__(self, kernel: Kernel):
- self._kernel = kernel
-
- async def clear_cache(self, request: ClearCacheCommand) -> None:
- del request
- from marimo._save.cache import CacheContext
- from marimo._save.loaders import BasePersistenceLoader
-
- ctx = get_context()
- saved = 0
- for obj in ctx.globals.values():
- if isinstance(obj, CacheContext):
- if isinstance(obj.loader, BasePersistenceLoader):
- obj.loader.clear()
-
- broadcast_notification(CacheClearedNotification(bytes_freed=saved))
-
- async def get_cache_info(self, request: GetCacheInfoCommand) -> None:
- del request
- from marimo._save.cache import CacheContext
-
- ctx = get_context()
- total_hits = 0
- total_misses = 0
- total_time = 0
- disk_to_free = -1 # TODO: sum up disk usage
- disk_total = -1
-
- for obj in ctx.globals.values():
- if isinstance(obj, CacheContext):
- hits, misses, _, _, time = obj.cache_info()
- total_hits += hits
- total_misses += misses
- total_time += time
- # d2f, dt = obj.loader.disk_usage()
- broadcast_notification(
- CacheInfoNotification(
- hits=total_hits,
- misses=total_misses,
- time=total_time,
- disk_to_free=disk_to_free,
- disk_total=disk_total,
- ),
- )
-
-
-class RequestHandler:
- def __init__(self) -> None:
- self._handlers: dict[
- type[CommandMessage],
- Callable[[CommandMessage], Awaitable[None]],
- ] = {}
-
- def register(
- self,
- request_type: type[CommandMessage],
- handler: Callable[[Any], Awaitable[None]],
- ) -> None:
- self._handlers[request_type] = handler
-
- async def handle(self, request: CommandMessage) -> None:
- handler = self._handlers.get(type(request))
- if handler:
- return await handler(request)
- raise ValueError(f"Unknown request {request}")
-
-
@dataclasses.dataclass
class _LaunchStreams:
"""Superset of `KernelStreams` carrying the subprocess bootstrap pieces."""
diff --git a/tests/_runtime/test_manage_script_metadata.py b/tests/_runtime/test_manage_script_metadata.py
index f78bb03f0d..ae00199ac5 100644
--- a/tests/_runtime/test_manage_script_metadata.py
+++ b/tests/_runtime/test_manage_script_metadata.py
@@ -354,7 +354,8 @@ async def test_missing_packages_hook(
with (
patch(
- "marimo._runtime.runtime.broadcast_notification", mock_broadcast
+ "marimo._runtime.callbacks.packages.broadcast_notification",
+ mock_broadcast,
),
patch("micropip.install", new_callable=AsyncMock),
):
@@ -448,7 +449,8 @@ def test_missing_packages_hook_pip(
with (
patch(
- "marimo._runtime.runtime.broadcast_notification", mock_broadcast
+ "marimo._runtime.callbacks.packages.broadcast_notification",
+ mock_broadcast,
),
):
k.packages_callbacks.package_manager = create_package_manager("pip")
@@ -532,7 +534,8 @@ async def test_install_missing_packages_with_streaming_logs(
with (
patch(
- "marimo._runtime.runtime.broadcast_notification", mock_broadcast
+ "marimo._runtime.callbacks.packages.broadcast_notification",
+ mock_broadcast,
),
):
# Create install request
@@ -605,7 +608,8 @@ async def test_install_missing_packages_streaming_logs_failure(
with (
patch(
- "marimo._runtime.runtime.broadcast_notification", mock_broadcast
+ "marimo._runtime.callbacks.packages.broadcast_notification",
+ mock_broadcast,
),
):
request = InstallPackagesCommand(
@@ -668,7 +672,8 @@ async def test_install_missing_packages_streaming_logs_multiple_packages(
with (
patch(
- "marimo._runtime.runtime.broadcast_notification", mock_broadcast
+ "marimo._runtime.callbacks.packages.broadcast_notification",
+ mock_broadcast,
),
):
request = InstallPackagesCommand(
@@ -742,7 +747,8 @@ async def test_install_missing_packages_no_logs_backward_compatibility(
with (
patch(
- "marimo._runtime.runtime.broadcast_notification", mock_broadcast
+ "marimo._runtime.callbacks.packages.broadcast_notification",
+ mock_broadcast,
),
):
request = InstallPackagesCommand(
diff --git a/tests/_runtime/test_request_router.py b/tests/_runtime/test_request_router.py
new file mode 100644
index 0000000000..0e53b3492b
--- /dev/null
+++ b/tests/_runtime/test_request_router.py
@@ -0,0 +1,166 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import typing
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from marimo._runtime.callbacks import (
+ CacheCallbacks,
+ DatasetCallbacks,
+ ExternalStorageCallbacks,
+ KernelCallback,
+ PackagesCallbacks,
+ SecretsCallbacks,
+ SqlCallbacks,
+)
+from marimo._runtime.commands import (
+ ClearCacheCommand,
+ CodeCompletionCommand,
+ CommandMessage,
+ StopKernelCommand,
+)
+from marimo._runtime.kernel_request_handlers import KernelRequestHandlers
+from marimo._runtime.request_router import RequestRouter
+
+if TYPE_CHECKING:
+ from tests.conftest import MockedKernel
+
+
+# All callback bundles that self-register against a router. Order doesn't
+# matter — tests only iterate this list, never index into it.
+ALL_CALLBACKS: list[type] = [
+ SecretsCallbacks,
+ DatasetCallbacks,
+ SqlCallbacks,
+ CacheCallbacks,
+ ExternalStorageCallbacks,
+ PackagesCallbacks,
+]
+
+# Commands that are part of the CommandMessage dispatch surface but are
+# intentionally not handled by the kernel's RequestRouter. CodeCompletionCommand
+# is delivered on its own queue and processed by start_completion_worker.
+NOT_ROUTED: set[type] = {CodeCompletionCommand}
+
+
+def _all_command_classes() -> set[type]:
+ """Every concrete command type in the CommandMessage discriminated union."""
+ return set(typing.get_args(CommandMessage))
+
+
+class TestRequestRouter:
+ async def test_dispatch_invokes_registered_handler(self) -> None:
+ router = RequestRouter()
+ seen: list[Any] = []
+
+ async def handler(request: StopKernelCommand) -> None:
+ seen.append(request)
+
+ router.register(StopKernelCommand, handler)
+ cmd = StopKernelCommand()
+ await router.dispatch(cmd)
+ assert seen == [cmd]
+
+ async def test_dispatch_unknown_command_raises(self) -> None:
+ router = RequestRouter()
+ with pytest.raises(ValueError, match="Unknown request"):
+ await router.dispatch(StopKernelCommand())
+
+ async def test_register_overwrites_prior_binding(self) -> None:
+ router = RequestRouter()
+ calls: list[str] = []
+
+ async def first(_: StopKernelCommand) -> None:
+ calls.append("first")
+
+ async def second(_: StopKernelCommand) -> None:
+ calls.append("second")
+
+ router.register(StopKernelCommand, first)
+ router.register(StopKernelCommand, second)
+ await router.dispatch(StopKernelCommand())
+ assert calls == ["second"]
+
+ async def test_dispatch_routes_by_exact_type(self) -> None:
+ router = RequestRouter()
+ seen: list[str] = []
+
+ async def stop_handler(_: StopKernelCommand) -> None:
+ seen.append("stop")
+
+ async def clear_handler(_: ClearCacheCommand) -> None:
+ seen.append("clear")
+
+ router.register(StopKernelCommand, stop_handler)
+ router.register(ClearCacheCommand, clear_handler)
+ await router.dispatch(ClearCacheCommand())
+ await router.dispatch(StopKernelCommand())
+ assert seen == ["clear", "stop"]
+
+
+ "callback_cls", ALL_CALLBACKS, ids=lambda c: c.__name__
+)
+def test_callback_implements_kernel_callback_protocol(
+ callback_cls: type,
+ mocked_kernel: MockedKernel,
+) -> None:
+ assert isinstance(callback_cls(mocked_kernel.k), KernelCallback)
+
+
+def test_no_two_callbacks_claim_the_same_command(
+ mocked_kernel: MockedKernel,
+) -> None:
+ """Each callback bundle must own a disjoint slice of the command surface."""
+ owner: dict[type, str] = {}
+ for cb_cls in ALL_CALLBACKS:
+ router = RequestRouter()
+ cb_cls(mocked_kernel.k).register(router)
+ for cmd in router._handlers:
+ assert cmd not in owner, (
+ f"{cmd.__name__} is registered by both "
+ f"{owner[cmd]} and {cb_cls.__name__}"
+ )
+ owner[cmd] = cb_cls.__name__
+
+
+def test_kernel_handlers_and_callbacks_partition_the_command_surface(
+ mocked_kernel: MockedKernel,
+) -> None:
+ """Kernel-owned handlers and callback-owned handlers must not overlap."""
+ kernel_router = RequestRouter()
+ KernelRequestHandlers(mocked_kernel.k).register(kernel_router)
+ kernel_bound = set(kernel_router._handlers)
+
+ callback_bound: set[type] = set()
+ for cb_cls in ALL_CALLBACKS:
+ router = RequestRouter()
+ cb_cls(mocked_kernel.k).register(router)
+ callback_bound |= set(router._handlers)
+
+ overlap = kernel_bound & callback_bound
+ assert not overlap, (
+ f"commands handled by both KernelRequestHandlers and a callback: "
+ f"{sorted(c.__name__ for c in overlap)}"
+ )
+
+
+def test_kernel_router_covers_every_dispatchable_command(
+ mocked_kernel: MockedKernel,
+) -> None:
+ """Every CommandMessage member (minus NOT_ROUTED) has a handler."""
+ bound = set(mocked_kernel.k.router._handlers)
+ expected = _all_command_classes() - NOT_ROUTED
+
+ missing = expected - bound
+ extra = bound - expected
+ assert not missing, (
+ f"commands with no handler on the kernel router: "
+ f"{sorted(c.__name__ for c in missing)}"
+ )
+ assert not extra, (
+ f"handlers bound for commands not in CommandMessage: "
+ f"{sorted(c.__name__ for c in extra)}"
+ )
diff --git a/tests/_runtime/test_runtime.py b/tests/_runtime/test_runtime.py
index 537b529735..d6cbbc711e 100644
--- a/tests/_runtime/test_runtime.py
+++ b/tests/_runtime/test_runtime.py
@@ -4067,24 +4067,6 @@ def _filter_to_error_ops(
]
-class TestRequestHandler:
- async def test_request_handler_only_created_once(
- self, any_kernel: Kernel
- ) -> None:
- """Test that request_handler property is only created once."""
- k = any_kernel
-
- # Access request_handler multiple times
- handler1 = k.request_handler
- handler2 = k.request_handler
- handler3 = k.request_handler
-
- # They should all be the same instance
- assert handler1 is handler2
- assert handler2 is handler3
- assert handler1 is handler3
-
-
class TestLaunchKernelEventLoop:
"""Event-loop policy / factory selection in launch_kernel.
diff --git a/tests/_runtime/test_runtime_secrets.py b/tests/_runtime/test_runtime_secrets.py
index bdb209e42e..7d4cbc70b5 100644
--- a/tests/_runtime/test_runtime_secrets.py
+++ b/tests/_runtime/test_runtime_secrets.py
@@ -6,11 +6,11 @@ from unittest.mock import MagicMock
import pytest
from marimo._messaging.notification import SecretKeysResultNotification
+from marimo._runtime.callbacks import SecretsCallbacks
from marimo._runtime.commands import (
ListSecretKeysCommand,
RefreshSecretsCommand,
)
-from marimo._runtime.runtime import SecretsCallbacks
from marimo._types.ids import RequestId
from tests._messaging.mocks import MockStream
from tests.conftest import MockedKernel