A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(table): recover from transient "function not found" in app mode (#9867)
Kiran Gadhave311491505d01
diff --git a/frontend/src/plugins/core/__test__/registerReactComponent.test.ts b/frontend/src/plugins/core/__test__/registerReactComponent.test.tsindex 2acd609f66..a285a72294 100644--- a/frontend/src/plugins/core/__test__/registerReactComponent.test.ts+++ b/frontend/src/plugins/core/__test__/registerReactComponent.test.ts@@ -3,11 +3,20 @@import ReactDOM from "react-dom/client";import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";import { z } from "zod";+import { FUNCTIONS_REGISTRY } from "@/core/functions/FunctionRegistry";+import type { FunctionCallResultMessage } from "@/core/kernel/messages";+import { FunctionNotFoundError } from "@/utils/errors";import {isCustomMarimoElement,registerReactComponent,} from "../registerReactComponent";+vi.mock("@/core/functions/FunctionRegistry", () => ({+ FUNCTIONS_REGISTRY: { request: vi.fn() },+}));++const requestMock = vi.mocked(FUNCTIONS_REGISTRY.request);+// Each custom element name can only be registered once per jsdom window,// so we use a counter to generate unique tag names across tests.let tagCounter = 0;@@ -202,3 +211,90 @@ describe("connectedCallback - light DOM nesting detection", () => {outer.remove();});});++describe("plugin function: not-found recovery", () => {+ const NOT_FOUND: FunctionCallResultMessage = {+ function_call_id:+ "call-id" as FunctionCallResultMessage["function_call_id"],+ status: { code: "error", message: "Function not found" },+ found: false,+ return_value: null,+ };+ const OK: FunctionCallResultMessage = {+ function_call_id:+ "call-id" as FunctionCallResultMessage["function_call_id"],+ status: { code: "ok" },+ found: true,+ return_value: { ok: true },+ };++ // Mount a plugin exposing a single callable function and resolve the+ // function map that PluginSlot hands to render(), so tests can invoke the+ // request path directly.+ async function mountWithFunction(): Promise<+ (args: Record<string, unknown>) => Promise<unknown>+ > {+ const tag = uniqueTag("fn");+ let captured:+ | ((args: Record<string, unknown>) => Promise<unknown>)+ | undefined;+ registerReactComponent({+ tagName: tag,+ validator: z.any(),+ functions: { run: { input: z.any(), output: z.any() } },+ render: ({ functions }) => {+ captured = (functions as Record<string, typeof captured>).run;+ return null as never;+ },+ });++ const wrapper = document.createElement("marimo-ui-element");+ wrapper.setAttribute("object-id", "test-object-id");+ const el = document.createElement(tag);+ wrapper.append(el);+ document.body.append(wrapper);++ await vi.waitFor(() => expect(captured).toBeDefined());+ return captured as (args: Record<string, unknown>) => Promise<unknown>;+ }++ beforeEach(() => {+ requestMock.mockReset();+ });++ afterEach(() => {+ document.body.innerHTML = "";+ });++ test("retries on found:false and resolves once the function appears", async () => {+ const run = await mountWithFunction();+ requestMock.mockResolvedValueOnce(NOT_FOUND).mockResolvedValueOnce(OK);++ await expect(run({})).resolves.toEqual({ ok: true });+ expect(requestMock).toHaveBeenCalledTimes(2);+ });++ test("throws FunctionNotFoundError after retries are exhausted", async () => {+ const run = await mountWithFunction();+ requestMock.mockResolvedValue(NOT_FOUND);++ await expect(run({})).rejects.toBeInstanceOf(FunctionNotFoundError);+ // Initial attempt plus three retries.+ expect(requestMock).toHaveBeenCalledTimes(4);+ });++ test("does not retry once the function is found (found:true)", async () => {+ const run = await mountWithFunction();+ requestMock.mockResolvedValue({+ function_call_id:+ "call-id" as FunctionCallResultMessage["function_call_id"],+ status: { code: "error", message: "boom" },+ found: true,+ return_value: null,+ });++ const promise = run({});+ await expect(promise).rejects.toThrow("boom");+ expect(requestMock).toHaveBeenCalledTimes(1);+ });+});diff --git a/frontend/src/plugins/core/registerReactComponent.tsx b/frontend/src/plugins/core/registerReactComponent.tsxindex 52787372c9..60af78236a 100644--- a/frontend/src/plugins/core/registerReactComponent.tsx+++ b/frontend/src/plugins/core/registerReactComponent.tsx@@ -40,7 +40,10 @@ import {} from "@/hooks/useEventListener";import { StyleNamespace } from "@/theme/namespace";import { useTheme } from "@/theme/useTheme";-import { CellNotInitializedError } from "@/utils/errors.ts";+import {+ CellNotInitializedError,+ FunctionNotFoundError,+} from "@/utils/errors.ts";import { Functions } from "@/utils/functions";import { shallowCompare } from "@/utils/shallow-compare";import { defineCustomElement } from "../../core/dom/defineCustomElement";@@ -80,6 +83,12 @@ export interface IMarimoHTMLElement extends HTMLElement {rerender: () => void;}+// Bounded exponential backoff for re-issuing a request whose function the+// kernel reported as not found. Short enough to fail fast on a genuinely+// missing function, long enough to outlast a transient object-id desync.+const FUNCTION_NOT_FOUND_MAX_RETRIES = 3;+const FUNCTION_NOT_FOUND_BASE_DELAY_MS = 150;+interface PluginSlotProps<T> {hostElement: HTMLElement;plugin: IPlugin<T, unknown>;@@ -188,8 +197,6 @@ function PluginSlotInternal<T>(args.length <= 1,`Plugin functions only supports a single argument. Called ${key}`,);- const objectId = getUIElementObjectId(hostElement);- invariant(objectId, "Object ID should exist");const isStatic = isStaticNotebook();@@ -218,16 +225,41 @@ function PluginSlotInternal<T>(Logger.warn(`Cell ID ${cellId} cannot be found`);}- const response = await FUNCTIONS_REGISTRY.request({- args: prettyParse(input, args[0]),- functionName: key,- namespace: objectId,- });- if (response.status.code !== "ok") {+ // A "function not found" response means the kernel never ran the+ // function, so re-issuing the request is side-effect-safe regardless+ // of whether the function is idempotent. This recovers from a+ // transient window where the frontend's object-id leads the kernel's+ // registry. The object-id is re-read each attempt so a corrected id+ // from a re-render is picked up. Once the function is found+ // (found === true) the request is never retried, since the failure+ // is unrelated to lookup and retrying would not help.+ const parsedArgs = prettyParse(input, args[0]);+ for (let attempt = 0; ; attempt++) {+ const namespace = getUIElementObjectId(hostElement);+ invariant(namespace, "Object ID should exist");++ const response = await FUNCTIONS_REGISTRY.request({+ args: parsedArgs,+ functionName: key,+ namespace,+ });+ if (response.status.code === "ok") {+ return prettyParse(output, response.return_value);+ }++ const recoverable = response.found === false;+ if (recoverable && attempt < FUNCTION_NOT_FOUND_MAX_RETRIES) {+ const delay = FUNCTION_NOT_FOUND_BASE_DELAY_MS * 2 ** attempt;+ await new Promise((resolve) => setTimeout(resolve, delay));+ continue;+ }+Logger.error(response.status);+ if (recoverable) {+ throw new FunctionNotFoundError();+ }throw new Error(response.status.message || "Unknown error");}- return prettyParse(output, response.return_value);};}diff --git a/frontend/src/utils/errors.ts b/frontend/src/utils/errors.tsindex 8ef7b4512e..a89c0d2322 100644--- a/frontend/src/utils/errors.ts+++ b/frontend/src/utils/errors.ts@@ -71,6 +71,15 @@ export class CellNotInitializedError extends Error {}}+export class FunctionNotFoundError extends Error {+ constructor(+ message = "This UI element could not reach its function on the kernel. Try re-running the cell.",+ ) {+ super(message);+ this.name = "FunctionNotFoundError";+ }+}+export class NoKernelConnectedError extends Error {constructor(message = "Not yet connected to a kernel.") {super(message);diff --git a/marimo/_messaging/notification.py b/marimo/_messaging/notification.pyindex 0c80287b09..155f49e8d4 100644--- a/marimo/_messaging/notification.py+++ b/marimo/_messaging/notification.py@@ -131,6 +131,11 @@ class FunctionCallResultNotification(Notification, tag="function-call-result"):function_call_id: ID matching the original request.return_value: Function return value as JSON.status: Human-readable success/failure status.+ found: Whether the requested function was located in the registry.+ False signals a transient registry desync, so the request is safe+ to retry. True means no retry will help: a non-ok status then+ reflects a failure unrelated to lookup, such as the function+ raising during execution or not being associated with a cell."""name: ClassVar[str] = "function-call-result"@@ -138,6 +143,7 @@ class FunctionCallResultNotification(Notification, tag="function-call-result"):function_call_id: RequestIdreturn_value: JSONTypestatus: HumanReadableStatus+ found: boolclass RemoveUIElementsNotification(Notification, tag="remove-ui-elements"):diff --git a/marimo/_runtime/kernel_request_handlers.py b/marimo/_runtime/kernel_request_handlers.pyindex 78740bc8e1..c2cf3a69ba 100644--- a/marimo/_runtime/kernel_request_handlers.py+++ b/marimo/_runtime/kernel_request_handlers.py@@ -167,13 +167,14 @@ class KernelRequestHandlers:async def _handle_function_call(self, request: InvokeFunctionCommand) -> None:- status, ret, _ = await self._kernel.function_call_request(request)+ status, ret, found = await self._kernel.function_call_request(request)LOGGER.debug("Function returned with status %s", status)broadcast_notification(FunctionCallResultNotification(function_call_id=request.function_call_id,return_value=ret,status=status,+ found=found,),)diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.pyindex d3698892e9..6210227817 100644--- a/marimo/_runtime/runtime.py+++ b/marimo/_runtime/runtime.py@@ -2055,6 +2055,26 @@ class Kernel:error_title = "Function not found"error_message = f"Could not find function given request: {request}"debug(error_title, error_message)+ # Logged at warning level so the field trigger is visible in+ # production. The requested namespace missing from the registry is+ # the signature of a frontend/kernel object-id desync; the set of+ # registered namespaces lets us compare object-id (cell) prefixes+ # to tell an unknown cell apart from a stale one. Object-ids and+ # filenames only, never argument values.+ LOGGER.warning(+ "Function call not found "+ "(pid=%s, notebook=%s, namespace=%s, function=%s, "+ "namespace_registered=%s, registered_namespace_count=%d, "+ "registered_namespaces=%s, child_contexts_searched=%d)",+ os.getpid(),+ self.app_metadata.filename,+ request.namespace,+ request.function_name,+ request.namespace in ctx.function_registry.namespaces,+ len(ctx.function_registry.namespaces),+ sorted(ctx.function_registry.namespaces),+ sum(1 for child in ctx.children if child.app is not None),+ )elif function.cell_id is None:found = Trueerror_title = "Function not associated with cell"diff --git a/marimo/_schemas/generated/notifications.yaml b/marimo/_schemas/generated/notifications.yamlindex ed1d67798c..996064b1c0 100644--- a/marimo/_schemas/generated/notifications.yaml+++ b/marimo/_schemas/generated/notifications.yaml@@ -695,8 +695,15 @@ components:description: "Result of a frontend-initiated function call.\n\n Attributes:\n\\ function_call_id: ID matching the original request.\n return_value:\\ Function return value as JSON.\n status: Human-readable success/failure\- \ status."- properties:+ \ status.\n found: Whether the requested function was located in the\+ \ registry.\n False signals a transient registry desync, so the\+ \ request is safe\n to retry. True means no retry will help: a\+ \ non-ok status then\n reflects a failure unrelated to lookup,\+ \ such as the function\n raising during execution or not being\+ \ associated with a cell."+ properties:+ found:+ type: booleanfunction_call_id:type: stringop:@@ -710,6 +717,7 @@ components:- function_call_id- return_value- status+ - foundtitle: FunctionCallResultNotificationtype: objectHumanReadableStatus:diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yamlindex 02a162cf47..7b86d7e5b9 100644--- a/packages/openapi/api.yaml+++ b/packages/openapi/api.yaml@@ -2116,8 +2116,15 @@ components:description: "Result of a frontend-initiated function call.\n\n Attributes:\n\\ function_call_id: ID matching the original request.\n return_value:\\ Function return value as JSON.\n status: Human-readable success/failure\- \ status."- properties:+ \ status.\n found: Whether the requested function was located in the\+ \ registry.\n False signals a transient registry desync, so the\+ \ request is safe\n to retry. True means no retry will help: a\+ \ non-ok status then\n reflects a failure unrelated to lookup,\+ \ such as the function\n raising during execution or not being\+ \ associated with a cell."+ properties:+ found:+ type: booleanfunction_call_id:$ref: '#/components/schemas/RequestId'op:@@ -2131,6 +2138,7 @@ components:- function_call_id- return_value- status+ - foundtitle: FunctionCallResultNotificationtype: objectGetCacheInfoCommand:diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.tsindex 4e221a15b4..6d57f54158 100644--- a/packages/openapi/src/api.ts+++ b/packages/openapi/src/api.ts@@ -4660,8 +4660,14 @@ export interface components {* function_call_id: ID matching the original request.* return_value: Function return value as JSON.* status: Human-readable success/failure status.+ * found: Whether the requested function was located in the registry.+ * False signals a transient registry desync, so the request is safe+ * to retry. True means no retry will help: a non-ok status then+ * reflects a failure unrelated to lookup, such as the function+ * raising during execution or not being associated with a cell.*/FunctionCallResultNotification: {+ found: boolean;function_call_id: components["schemas"]["RequestId"];/** @enum {unknown} */op: "function-call-result";diff --git a/tests/_runtime/test_functions.py b/tests/_runtime/test_functions.pyindex 03cf1b4c2c..98afe7ab39 100644--- a/tests/_runtime/test_functions.py+++ b/tests/_runtime/test_functions.py@@ -1,12 +1,23 @@from __future__ import annotationsimport dataclasses+import logging+from typing import TYPE_CHECKING+from marimo import _loggers+from marimo._runtime.commands import InvokeFunctionCommand+from marimo._runtime.context import get_contextfrom marimo._runtime.functions import (Function,FunctionNamespace,FunctionRegistry,)+from marimo._types.ids import RequestId++if TYPE_CHECKING:+ import pytest++ from marimo._runtime.runtime import Kernel@dataclasses.dataclass@@ -84,3 +95,82 @@ def test_function_registry() -> None:registry.delete(namespace)assert registry.get_function(namespace, "test_function") is None+++def _invoke(namespace: str, function_name: str) -> InvokeFunctionCommand:+ return InvokeFunctionCommand(+ function_call_id=RequestId("call"),+ namespace=namespace,+ function_name=function_name,+ args={"value": 1},+ )+++async def test_function_call_request_not_found(k: Kernel) -> None:+ status, _, found = await k.function_call_request(+ _invoke("unregistered-namespace", "missing")+ )+ assert found is False+ assert status.code == "error"+ assert status.title == "Function not found"+++async def test_function_call_request_found_after_register(k: Kernel) -> None:+ namespace = "registered-namespace"+ get_context().function_registry.register(+ namespace,+ Function(name="echo", arg_cls=Args, function=lambda x: x.value),+ )++ _, _, found = await k.function_call_request(_invoke(namespace, "echo"))+ assert found is True+++async def test_function_call_request_not_found_logs(+ k: Kernel, caplog: pytest.LogCaptureFixture+) -> None:+ logger = _loggers.marimo_logger()+ old_propagate = logger.propagate+ logger.propagate = True+ try:+ with caplog.at_level(logging.WARNING):+ await k.function_call_request(_invoke("missing-ns", "missing-fn"))+ finally:+ logger.propagate = old_propagate++ records = [+ record+ for record in caplog.records+ if "Function call not found" in record.getMessage()+ ]+ assert len(records) == 1+ message = records[0].getMessage()+ assert "namespace=missing-ns" in message+ assert "function=missing-fn" in message+ assert "namespace_registered=False" in message+ assert "child_contexts_searched=0" in message+++async def test_function_call_request_found_does_not_log(+ k: Kernel, caplog: pytest.LogCaptureFixture+) -> None:+ namespace = "registered-namespace"+ get_context().function_registry.register(+ namespace,+ Function(name="echo", arg_cls=Args, function=lambda x: x.value),+ )++ logger = _loggers.marimo_logger()+ old_propagate = logger.propagate+ logger.propagate = True+ try:+ with caplog.at_level(logging.WARNING):+ await k.function_call_request(_invoke(namespace, "echo"))+ finally:+ logger.propagate = old_propagate++ assert not [+ record+ for record in caplog.records+ if "Function call not found" in record.getMessage()+ ]