A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
Replace pickle with msgspec for IPC serialization (#8713)
Trevor Manzdf4e40ccc33d
diff --git a/marimo/_ipc/connection.py b/marimo/_ipc/connection.pyindex 1e5da6bce4..7f65ee2337 100644--- a/marimo/_ipc/connection.py+++ b/marimo/_ipc/connection.py@@ -8,20 +8,22 @@ import queueimport sysimport typing+import msgspec+from marimo import _loggersfrom marimo._ipc.queue_proxy import PushQueue, start_receiver_threadfrom marimo._ipc.types import ConnectionInfo+from marimo._runtime.commands import (+ BatchableCommand,+ CodeCompletionCommand,+ CommandMessage,+)from marimo._session.queue import QueueTypeif typing.TYPE_CHECKING:import zmqfrom marimo._messaging.types import KernelMessage- from marimo._runtime.commands import (- BatchableCommand,- CodeCompletionCommand,- CommandMessage,- )LOGGER = _loggers.marimo_logger()ADDR = "tcp://127.0.0.1"@@ -41,6 +43,7 @@ class Channel(typing.Generic[T]):kind: typing.Literal["push", "pull"]socket: zmq.Socket[bytes]queue: QueueType[T]+ decoder: msgspec.msgpack.Decoder[T] | None = None@classmethoddef Push(@@ -61,12 +64,17 @@ class Channel(typing.Generic[T]):@classmethoddef Pull(- cls, context: zmq.Context[zmq.Socket[bytes]], *, maxsize: int = 0+ cls,+ context: zmq.Context[zmq.Socket[bytes]],+ *,+ msg_type: type[T],+ maxsize: int = 0,) -> Channel[T]:"""Create a pull (receive-only) channel.Args:context: ZeroMQ context for creating sockets+ msg_type: The type to decode incoming messages asmaxsize: Maximum queue size (0 = unlimited)"""import zmq@@ -76,6 +84,7 @@ class Channel(typing.Generic[T]):kind="pull",socket=socket,queue=queue.Queue(maxsize=maxsize),+ decoder=msgspec.msgpack.Decoder(msg_type),)@@ -95,22 +104,22 @@ class Connection:def __post_init__(self) -> None:"""Start receiver threads for all pull channels."""- receivers: dict[zmq.Socket[bytes], QueueType[typing.Any]] = {}+ pull_channels: list[Channel[typing.Any]] = []if self.control.kind == "pull":- receivers[self.control.socket] = self.control.queue+ pull_channels.append(self.control)if self.ui_element.kind == "pull":- receivers[self.ui_element.socket] = self.ui_element.queue+ pull_channels.append(self.ui_element)if self.completion.kind == "pull":- receivers[self.completion.socket] = self.completion.queue+ pull_channels.append(self.completion)if self.win32_interrupt and self.win32_interrupt.kind == "pull":- receivers[self.win32_interrupt.socket] = self.win32_interrupt.queue+ pull_channels.append(self.win32_interrupt)if self.input.kind == "pull":- receivers[self.input.socket] = self.input.queue+ pull_channels.append(self.input)if self.stream.kind == "pull":- receivers[self.stream.socket] = self.stream.queue+ pull_channels.append(self.stream)self._stop_event, self._receiver_thread = start_receiver_thread(- receivers+ pull_channels)@classmethod@@ -132,7 +141,7 @@ class Connection:Channel.Push(context) if sys.platform == "win32" else None),input=Channel.Push(context, maxsize=1),- stream=Channel.Pull(context),+ stream=Channel.Pull(context, msg_type=bytes),)info = ConnectionInfo(control=conn.control.socket.bind_to_random_port(ADDR),@@ -164,13 +173,13 @@ class Connection:conn = cls(context=context,- control=Channel.Pull(context),- ui_element=Channel.Pull(context),- completion=Channel.Pull(context),- win32_interrupt=Channel.Pull(context)+ control=Channel.Pull(context, msg_type=CommandMessage),+ ui_element=Channel.Pull(context, msg_type=BatchableCommand),+ completion=Channel.Pull(context, msg_type=CodeCompletionCommand),+ win32_interrupt=Channel.Pull(context, msg_type=bool)if connection_info.win32_interruptelse None,- input=Channel.Pull(context, maxsize=1),+ input=Channel.Pull(context, msg_type=str, maxsize=1),stream=Channel.Push(context),)diff --git a/marimo/_ipc/queue_proxy.py b/marimo/_ipc/queue_proxy.pyindex fff4a4aece..cede277b38 100644--- a/marimo/_ipc/queue_proxy.py+++ b/marimo/_ipc/queue_proxy.py@@ -3,10 +3,11 @@from __future__ import annotations-import pickleimport threadingimport typing+import msgspec+from marimo import _loggersfrom marimo._session.queue import QueueType@@ -17,6 +18,8 @@ T = typing.TypeVar("T")if typing.TYPE_CHECKING:import zmq+ from marimo._ipc.connection import Channel+class PushQueue(QueueType[T]):"""Queue for pushing messages through ZeroMQ socket (sender side only).@@ -40,7 +43,7 @@ class PushQueue(QueueType[T]):timeout: float | None = None, # noqa: ARG002) -> None:"""Put an item into the queue."""- self.socket.send(pickle.dumps(obj))+ self.socket.send(msgspec.msgpack.encode(obj))def put_nowait(self, obj: T) -> None:"""Put an item into the queue without blocking."""@@ -63,19 +66,21 @@ class PushQueue(QueueType[T]):def start_receiver_thread(- receivers: dict[zmq.Socket[bytes], QueueType[typing.Any]],+ channels: list[Channel[typing.Any]],) -> tuple[threading.Event, threading.Thread]:"""Start receiver thread."""import zmqdef receive_loop(- receivers: dict[zmq.Socket[bytes], QueueType[typing.Any]],+ channels: list[Channel[typing.Any]],stop_event: threading.Event,) -> None:"""Receive messages from sockets and put them in queues using polling."""poller = zmq.Poller()- for socket in receivers:- poller.register(socket, zmq.POLLIN)+ socket_to_channel: dict[zmq.Socket[bytes], Channel[typing.Any]] = {}+ for channel in channels:+ poller.register(channel.socket, zmq.POLLIN)+ socket_to_channel[channel.socket] = channelwhile not stop_event.is_set():try:@@ -83,9 +88,12 @@ def start_receiver_thread(socks = dict(poller.poll(100))for socket, event in socks.items():if event & zmq.POLLIN:+ ch = socket_to_channel[socket]msg = socket.recv(flags=zmq.NOBLOCK)- obj = pickle.loads(msg)- receivers[socket].put(obj)+ assert ch.decoder is not None, (+ "Pull channel must have a decoder"+ )+ ch.queue.put(ch.decoder.decode(msg))except zmq.Again:continueexcept zmq.ZMQError as e:@@ -101,7 +109,7 @@ def start_receiver_thread(stop_event = threading.Event()thread = threading.Thread(target=receive_loop,- args=(receivers, stop_event),+ args=(channels, stop_event),daemon=True,)thread.start()diff --git a/tests/_ipc/test_kernel_communication.py b/tests/_ipc/test_kernel_communication.pyindex 9fc4441b80..7accc48b15 100644--- a/tests/_ipc/test_kernel_communication.py+++ b/tests/_ipc/test_kernel_communication.py@@ -8,6 +8,7 @@ import subprocessimport sysimport time+import msgspecimport pytestfrom dirty_equals import IsFloat, IsList, IsUUID@@ -18,7 +19,14 @@ from marimo._config.settings import GLOBAL_SETTINGSfrom marimo._dependencies.dependencies import DependencyManagerfrom marimo._runtime.commands import (AppMetadata,+ BatchableCommand,+ CodeCompletionCommand,+ CommandMessage,ExecuteCellsCommand,+ HTTPRequest,+ ModelCommand,+ ModelUpdateMessage,+ UpdateUIElementCommand,)from marimo._types.ids import CellId_t@@ -259,9 +267,141 @@ def test_queue_manager_connection():host_manager.control_queue.put(test_request)assert client_manager.control_queue.get(timeout=1) == test_request- kernel_message = ("test-op", b'{"data": "test"}')+ kernel_message = b'{"op": "test-op", "data": "test"}'client_manager.stream_queue.put(kernel_message)assert host_manager.stream_queue.get(timeout=1) == kernel_messagehost_manager.close_queues()client_manager.close_queues()+++class TestMsgpackIPC:+ """Test msgspec msgpack serialization for IPC channels.++ Each IPC channel has a specific type that must survive encode/decode.+ These tests mirror the channel types in Connection.create/connect:++ control: CommandMessage (discriminated union)+ ui_element: BatchableCommand (discriminated union)+ completion: CodeCompletionCommand+ input: str+ win32_interrupt: bool+ stream: bytes+ """++ def test_control_channel(self) -> None:+ """CommandMessage union dispatches to the correct subtype."""+ cmd = ExecuteCellsCommand(cell_ids=["c1"], codes=["x=1"])+ encoded = msgspec.msgpack.encode(cmd)+ decoded = msgspec.msgpack.Decoder(CommandMessage).decode(encoded)+ assert decoded == cmd++ def test_control_channel_with_http_request(self) -> None:+ """HTTPRequest (a @dataclass) survives the round-trip as a nested field."""+ req = HTTPRequest(+ url={"path": "/run"},+ base_url={"path": "/"},+ headers={"content-type": "application/json"},+ query_params={"key": ["val1", "val2"]},+ path_params={},+ cookies={},+ meta={},+ user={},+ )+ cmd = ExecuteCellsCommand(cell_ids=["c1"], codes=["x=1"], request=req)+ encoded = msgspec.msgpack.encode(cmd)+ decoded = msgspec.msgpack.Decoder(CommandMessage).decode(encoded)+ assert decoded.request is not None+ assert decoded.request["url"]["path"] == "/run"+ assert decoded.request["query_params"]["key"] == ["val1", "val2"]++ def test_ui_element_channel(self) -> None:+ """BatchableCommand union round-trips both member types."""+ ui = UpdateUIElementCommand(+ object_ids=["e1"], values=[{"nested": [1, 2]}], token="tok"+ )+ encoded = msgspec.msgpack.encode(ui)+ assert msgspec.msgpack.Decoder(BatchableCommand).decode(encoded) == ui++ msg = ModelUpdateMessage(state={"k": "v"}, buffer_paths=[])+ model = ModelCommand(+ model_id="m1", message=msg, buffers=[b"buf"], token="tok"+ )+ encoded = msgspec.msgpack.encode(model)+ assert (+ msgspec.msgpack.Decoder(BatchableCommand).decode(encoded) == model+ )++ def test_completion_channel(self) -> None:+ cmd = CodeCompletionCommand(id="t", document="x = 1", cell_id="c1")+ encoded = msgspec.msgpack.encode(cmd)+ decoded = msgspec.msgpack.Decoder(CodeCompletionCommand).decode(+ encoded+ )+ assert decoded == cmd++ def test_primitive_channels(self) -> None:+ """str (input), bool (win32_interrupt), and bytes (stream) channels."""+ for value, typ in [+ ("user_input", str),+ (True, bool),+ (b'{"op": "cell-op"}', bytes),+ ]:+ encoded = msgspec.msgpack.encode(value)+ assert msgspec.msgpack.Decoder(typ).decode(encoded) == value++ def test_unknown_fields_are_ignored(self) -> None:+ """Decoder silently drops fields it doesn't recognize.++ This matters when the sender is newer than the receiver (e.g.+ a field was added to a command). msgspec must not reject the+ message — it should decode the known fields and discard the rest.+ """++ # A "V2" struct with the same tag but an extra field+ class ExecuteCellsCommandV2(+ msgspec.Struct,+ rename="camel",+ tag_field="type",+ tag="execute-cells",+ ):+ cell_ids: list[str]+ codes: list[str]+ new_field: str = "added_later"++ v2 = ExecuteCellsCommandV2(+ cell_ids=["c1"], codes=["x=1"], new_field="extra"+ )+ encoded = msgspec.msgpack.encode(v2)++ decoded = msgspec.msgpack.Decoder(CommandMessage).decode(encoded)+ assert type(decoded) is ExecuteCellsCommand+ assert decoded.cell_ids == ["c1"]+ assert decoded.codes == ["x=1"]++ def test_missing_optional_fields_get_defaults(self) -> None:+ """Decoder fills in defaults for fields the sender didn't include.++ This matters when the receiver is newer than the sender (e.g.+ a field was added with a default, but the sender hasn't been+ updated yet).+ """++ class ExecuteCellsCommandV2(+ msgspec.Struct,+ rename="camel",+ tag_field="type",+ tag="execute-cells",+ ):+ cell_ids: list[str]+ codes: list[str]+ new_field: str = "default_value"++ old = ExecuteCellsCommand(cell_ids=["c1"], codes=["x=1"])+ encoded = msgspec.msgpack.encode(old)++ decoded = msgspec.msgpack.Decoder(ExecuteCellsCommandV2).decode(+ encoded+ )+ assert decoded.cell_ids == ["c1"]+ assert decoded.new_field == "default_value"