Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

Replace pickle with msgspec for IPC serialization (#8713)

Trevor Manzdf4e40ccc33d
diff --git a/marimo/_ipc/connection.py b/marimo/_ipc/connection.py
index 1e5da6bce4..7f65ee2337 100644
--- a/marimo/_ipc/connection.py
+++ b/marimo/_ipc/connection.py
@@ -8,20 +8,22 @@ import queue
import sys
import typing
+import msgspec
+
from marimo import _loggers
from marimo._ipc.queue_proxy import PushQueue, start_receiver_thread
from marimo._ipc.types import ConnectionInfo
+from marimo._runtime.commands import (
+ BatchableCommand,
+ CodeCompletionCommand,
+ CommandMessage,
+)
from marimo._session.queue import QueueType
if typing.TYPE_CHECKING:
import zmq
from marimo._messaging.types import KernelMessage
- from marimo._runtime.commands import (
- BatchableCommand,
- CodeCompletionCommand,
- CommandMessage,
- )
LOGGER = _loggers.marimo_logger()
ADDR = "tcp://127.0.0.1"
@@ -41,6 +43,7 @@ class Channel(typing.Generic[T]):
kind: typing.Literal["push", "pull"]
socket: zmq.Socket[bytes]
queue: QueueType[T]
+ decoder: msgspec.msgpack.Decoder[T] | None = None
@classmethod
def Push(
@@ -61,12 +64,17 @@ class Channel(typing.Generic[T]):
@classmethod
def Pull(
- cls, context: zmq.Context[zmq.Socket[bytes]], *, maxsize: int = 0
+ cls,
+ context: zmq.Context[zmq.Socket[bytes]],
+ *,
+ msg_type: type[T],
+ maxsize: int = 0,
) -> Channel[T]:
"""Create a pull (receive-only) channel.
Args:
context: ZeroMQ context for creating sockets
+ msg_type: The type to decode incoming messages as
maxsize: Maximum queue size (0 = unlimited)
"""
import zmq
@@ -76,6 +84,7 @@ class Channel(typing.Generic[T]):
kind="pull",
socket=socket,
queue=queue.Queue(maxsize=maxsize),
+ decoder=msgspec.msgpack.Decoder(msg_type),
)
@@ -95,22 +104,22 @@ class Connection:
def __post_init__(self) -> None:
"""Start receiver threads for all pull channels."""
- receivers: dict[zmq.Socket[bytes], QueueType[typing.Any]] = {}
+ pull_channels: list[Channel[typing.Any]] = []
if self.control.kind == "pull":
- receivers[self.control.socket] = self.control.queue
+ pull_channels.append(self.control)
if self.ui_element.kind == "pull":
- receivers[self.ui_element.socket] = self.ui_element.queue
+ pull_channels.append(self.ui_element)
if self.completion.kind == "pull":
- receivers[self.completion.socket] = self.completion.queue
+ pull_channels.append(self.completion)
if self.win32_interrupt and self.win32_interrupt.kind == "pull":
- receivers[self.win32_interrupt.socket] = self.win32_interrupt.queue
+ pull_channels.append(self.win32_interrupt)
if self.input.kind == "pull":
- receivers[self.input.socket] = self.input.queue
+ pull_channels.append(self.input)
if self.stream.kind == "pull":
- receivers[self.stream.socket] = self.stream.queue
+ pull_channels.append(self.stream)
self._stop_event, self._receiver_thread = start_receiver_thread(
- receivers
+ pull_channels
)
@classmethod
@@ -132,7 +141,7 @@ class Connection:
Channel.Push(context) if sys.platform == "win32" else None
),
input=Channel.Push(context, maxsize=1),
- stream=Channel.Pull(context),
+ stream=Channel.Pull(context, msg_type=bytes),
)
info = ConnectionInfo(
control=conn.control.socket.bind_to_random_port(ADDR),
@@ -164,13 +173,13 @@ class Connection:
conn = cls(
context=context,
- control=Channel.Pull(context),
- ui_element=Channel.Pull(context),
- completion=Channel.Pull(context),
- win32_interrupt=Channel.Pull(context)
+ control=Channel.Pull(context, msg_type=CommandMessage),
+ ui_element=Channel.Pull(context, msg_type=BatchableCommand),
+ completion=Channel.Pull(context, msg_type=CodeCompletionCommand),
+ win32_interrupt=Channel.Pull(context, msg_type=bool)
if connection_info.win32_interrupt
else None,
- input=Channel.Pull(context, maxsize=1),
+ input=Channel.Pull(context, msg_type=str, maxsize=1),
stream=Channel.Push(context),
)
diff --git a/marimo/_ipc/queue_proxy.py b/marimo/_ipc/queue_proxy.py
index fff4a4aece..cede277b38 100644
--- a/marimo/_ipc/queue_proxy.py
+++ b/marimo/_ipc/queue_proxy.py
@@ -3,10 +3,11 @@
from __future__ import annotations
-import pickle
import threading
import typing
+import msgspec
+
from marimo import _loggers
from marimo._session.queue import QueueType
@@ -17,6 +18,8 @@ T = typing.TypeVar("T")
if typing.TYPE_CHECKING:
import zmq
+ from marimo._ipc.connection import Channel
+
class PushQueue(QueueType[T]):
"""Queue for pushing messages through ZeroMQ socket (sender side only).
@@ -40,7 +43,7 @@ class PushQueue(QueueType[T]):
timeout: float | None = None, # noqa: ARG002
) -> None:
"""Put an item into the queue."""
- self.socket.send(pickle.dumps(obj))
+ self.socket.send(msgspec.msgpack.encode(obj))
def put_nowait(self, obj: T) -> None:
"""Put an item into the queue without blocking."""
@@ -63,19 +66,21 @@ class PushQueue(QueueType[T]):
def start_receiver_thread(
- receivers: dict[zmq.Socket[bytes], QueueType[typing.Any]],
+ channels: list[Channel[typing.Any]],
) -> tuple[threading.Event, threading.Thread]:
"""Start receiver thread."""
import zmq
def receive_loop(
- receivers: dict[zmq.Socket[bytes], QueueType[typing.Any]],
+ channels: list[Channel[typing.Any]],
stop_event: threading.Event,
) -> None:
"""Receive messages from sockets and put them in queues using polling."""
poller = zmq.Poller()
- for socket in receivers:
- poller.register(socket, zmq.POLLIN)
+ socket_to_channel: dict[zmq.Socket[bytes], Channel[typing.Any]] = {}
+ for channel in channels:
+ poller.register(channel.socket, zmq.POLLIN)
+ socket_to_channel[channel.socket] = channel
while not stop_event.is_set():
try:
@@ -83,9 +88,12 @@ def start_receiver_thread(
socks = dict(poller.poll(100))
for socket, event in socks.items():
if event & zmq.POLLIN:
+ ch = socket_to_channel[socket]
msg = socket.recv(flags=zmq.NOBLOCK)
- obj = pickle.loads(msg)
- receivers[socket].put(obj)
+ assert ch.decoder is not None, (
+ "Pull channel must have a decoder"
+ )
+ ch.queue.put(ch.decoder.decode(msg))
except zmq.Again:
continue
except zmq.ZMQError as e:
@@ -101,7 +109,7 @@ def start_receiver_thread(
stop_event = threading.Event()
thread = threading.Thread(
target=receive_loop,
- args=(receivers, stop_event),
+ args=(channels, stop_event),
daemon=True,
)
thread.start()
diff --git a/tests/_ipc/test_kernel_communication.py b/tests/_ipc/test_kernel_communication.py
index 9fc4441b80..7accc48b15 100644
--- a/tests/_ipc/test_kernel_communication.py
+++ b/tests/_ipc/test_kernel_communication.py
@@ -8,6 +8,7 @@ import subprocess
import sys
import time
+import msgspec
import pytest
from dirty_equals import IsFloat, IsList, IsUUID
@@ -18,7 +19,14 @@ from marimo._config.settings import GLOBAL_SETTINGS
from marimo._dependencies.dependencies import DependencyManager
from marimo._runtime.commands import (
AppMetadata,
+ BatchableCommand,
+ CodeCompletionCommand,
+ CommandMessage,
ExecuteCellsCommand,
+ HTTPRequest,
+ ModelCommand,
+ ModelUpdateMessage,
+ UpdateUIElementCommand,
)
from marimo._types.ids import CellId_t
@@ -259,9 +267,141 @@ def test_queue_manager_connection():
host_manager.control_queue.put(test_request)
assert client_manager.control_queue.get(timeout=1) == test_request
- kernel_message = ("test-op", b'{"data": "test"}')
+ kernel_message = b'{"op": "test-op", "data": "test"}'
client_manager.stream_queue.put(kernel_message)
assert host_manager.stream_queue.get(timeout=1) == kernel_message
host_manager.close_queues()
client_manager.close_queues()
+
+
+class TestMsgpackIPC:
+ """Test msgspec msgpack serialization for IPC channels.
+
+ Each IPC channel has a specific type that must survive encode/decode.
+ These tests mirror the channel types in Connection.create/connect:
+
+ control: CommandMessage (discriminated union)
+ ui_element: BatchableCommand (discriminated union)
+ completion: CodeCompletionCommand
+ input: str
+ win32_interrupt: bool
+ stream: bytes
+ """
+
+ def test_control_channel(self) -> None:
+ """CommandMessage union dispatches to the correct subtype."""
+ cmd = ExecuteCellsCommand(cell_ids=["c1"], codes=["x=1"])
+ encoded = msgspec.msgpack.encode(cmd)
+ decoded = msgspec.msgpack.Decoder(CommandMessage).decode(encoded)
+ assert decoded == cmd
+
+ def test_control_channel_with_http_request(self) -> None:
+ """HTTPRequest (a @dataclass) survives the round-trip as a nested field."""
+ req = HTTPRequest(
+ url={"path": "/run"},
+ base_url={"path": "/"},
+ headers={"content-type": "application/json"},
+ query_params={"key": ["val1", "val2"]},
+ path_params={},
+ cookies={},
+ meta={},
+ user={},
+ )
+ cmd = ExecuteCellsCommand(cell_ids=["c1"], codes=["x=1"], request=req)
+ encoded = msgspec.msgpack.encode(cmd)
+ decoded = msgspec.msgpack.Decoder(CommandMessage).decode(encoded)
+ assert decoded.request is not None
+ assert decoded.request["url"]["path"] == "/run"
+ assert decoded.request["query_params"]["key"] == ["val1", "val2"]
+
+ def test_ui_element_channel(self) -> None:
+ """BatchableCommand union round-trips both member types."""
+ ui = UpdateUIElementCommand(
+ object_ids=["e1"], values=[{"nested": [1, 2]}], token="tok"
+ )
+ encoded = msgspec.msgpack.encode(ui)
+ assert msgspec.msgpack.Decoder(BatchableCommand).decode(encoded) == ui
+
+ msg = ModelUpdateMessage(state={"k": "v"}, buffer_paths=[])
+ model = ModelCommand(
+ model_id="m1", message=msg, buffers=[b"buf"], token="tok"
+ )
+ encoded = msgspec.msgpack.encode(model)
+ assert (
+ msgspec.msgpack.Decoder(BatchableCommand).decode(encoded) == model
+ )
+
+ def test_completion_channel(self) -> None:
+ cmd = CodeCompletionCommand(id="t", document="x = 1", cell_id="c1")
+ encoded = msgspec.msgpack.encode(cmd)
+ decoded = msgspec.msgpack.Decoder(CodeCompletionCommand).decode(
+ encoded
+ )
+ assert decoded == cmd
+
+ def test_primitive_channels(self) -> None:
+ """str (input), bool (win32_interrupt), and bytes (stream) channels."""
+ for value, typ in [
+ ("user_input", str),
+ (True, bool),
+ (b'{"op": "cell-op"}', bytes),
+ ]:
+ encoded = msgspec.msgpack.encode(value)
+ assert msgspec.msgpack.Decoder(typ).decode(encoded) == value
+
+ def test_unknown_fields_are_ignored(self) -> None:
+ """Decoder silently drops fields it doesn't recognize.
+
+ This matters when the sender is newer than the receiver (e.g.
+ a field was added to a command). msgspec must not reject the
+ message — it should decode the known fields and discard the rest.
+ """
+
+ # A "V2" struct with the same tag but an extra field
+ class ExecuteCellsCommandV2(
+ msgspec.Struct,
+ rename="camel",
+ tag_field="type",
+ tag="execute-cells",
+ ):
+ cell_ids: list[str]
+ codes: list[str]
+ new_field: str = "added_later"
+
+ v2 = ExecuteCellsCommandV2(
+ cell_ids=["c1"], codes=["x=1"], new_field="extra"
+ )
+ encoded = msgspec.msgpack.encode(v2)
+
+ decoded = msgspec.msgpack.Decoder(CommandMessage).decode(encoded)
+ assert type(decoded) is ExecuteCellsCommand
+ assert decoded.cell_ids == ["c1"]
+ assert decoded.codes == ["x=1"]
+
+ def test_missing_optional_fields_get_defaults(self) -> None:
+ """Decoder fills in defaults for fields the sender didn't include.
+
+ This matters when the receiver is newer than the sender (e.g.
+ a field was added with a default, but the sender hasn't been
+ updated yet).
+ """
+
+ class ExecuteCellsCommandV2(
+ msgspec.Struct,
+ rename="camel",
+ tag_field="type",
+ tag="execute-cells",
+ ):
+ cell_ids: list[str]
+ codes: list[str]
+ new_field: str = "default_value"
+
+ old = ExecuteCellsCommand(cell_ids=["c1"], codes=["x=1"])
+ encoded = msgspec.msgpack.encode(old)
+
+ decoded = msgspec.msgpack.Decoder(ExecuteCellsCommandV2).decode(
+ encoded
+ )
+ assert decoded.cell_ids == ["c1"]
+ assert decoded.new_field == "default_value"