Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

12.9 KB
Raw Blame
"""Tests for ZeroMQ-based kernel communication."""

from __future__ import annotations

import json
import queue
import subprocess
import sys
import time

import msgspec
import pytest
from dirty_equals import IsFloat, IsList, IsUUID

from marimo._ast.app_config import _AppConfig
from marimo._ast.cell import CellConfig
from marimo._config.config import DEFAULT_CONFIG
from marimo._config.settings import GLOBAL_SETTINGS
from marimo._dependencies.dependencies import DependencyManager
from marimo._runtime.commands import (
    AppMetadata,
    BatchableCommand,
    CodeCompletionCommand,
    CommandMessage,
    ExecuteCellsCommand,
    HTTPRequest,
    ModelCommand,
    ModelUpdateMessage,
    UpdateUIElementCommand,
)
from marimo._types.ids import CellId_t

HAS_DEPS = DependencyManager.has("zmq")


@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")
@pytest.mark.skip(reason="TODO: fix this test. Currently flaky on CI.")
def test_kernel_launch_and_execute_cells():
    """Test launching a kernel and executing cells with stdout/stderr."""
    from marimo._ipc import KernelArgs, QueueManager

    execute_request = ExecuteCellsCommand(
        cell_ids=[CellId_t("cell1")],
        codes=[
            """\
import sys
print("stdout")
print("stderr", file=sys.stderr)
x = 42"""
        ],
    )

    queue_manager, connection_info = QueueManager.create()
    kernel_args = KernelArgs(
        connection_info=connection_info,
        profile_path=None,
        configs={cid: CellConfig() for cid in execute_request.cell_ids},
        user_config=DEFAULT_CONFIG,
        log_level=GLOBAL_SETTINGS.LOG_LEVEL,
        app_metadata=AppMetadata(
            query_params={}, cli_args={}, app_config=_AppConfig()
        ),
    )

    # IMPORTANT: The module path "marimo._ipc.launch_kernel" is a public API
    # used by external consumers (e.g., marimo-lsp). Changing this path is a
    # BREAKING CHANGE and should be done with care and proper deprecation.
    process = subprocess.Popen(
        [sys.executable, "-m", "marimo._ipc.launch_kernel"],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )

    assert process.stdin is not None
    process.stdin.write(kernel_args.encode_json())
    process.stdin.flush()
    process.stdin.close()

    assert process.stdout is not None
    assert process.stderr is not None

    ready_line = process.stdout.readline().decode("utf-8").strip()

    if ready_line != "KERNEL_READY":
        exit_code = process.poll()
        stderr_content = process.stderr.read().decode("utf-8")
        if exit_code is not None and exit_code != 0:
            raise RuntimeError(
                f"Kernel process failed with exit code {exit_code}. Stderr: {stderr_content}"
            )
        else:
            raise RuntimeError(
                f"Expected KERNEL_READY, got: '{ready_line}'. Stderr: {stderr_content}"
            )

    queue_manager.control_queue.put(execute_request)

    messages = []
    seen_completed = False
    extra_collection_start = None

    while True:
        try:
            encoded = queue_manager.stream_queue.get(timeout=0.01)
            decoded = json.loads(encoded)
            messages.append(decoded)

            if decoded["op"] == "completed-run":
                seen_completed = True
                extra_collection_start = time.time()

        except queue.Empty:
            if seen_completed and extra_collection_start is not None:
                # FIXME: stdin/stdout are flushed every 10ms, so wait 100ms
                # (after "completed-run") to ensure all related events.
                if time.time() - extra_collection_start >= 0.1:
                    break

            # If we haven't seen completed-run yet, continue waiting
            continue

    assert messages == [
        {
            "op": "variables",
            "variables": IsList(
                {
                    "declared_by": [
                        "cell1",
                    ],
                    "name": "x",
                    "used_by": [],
                },
                {
                    "declared_by": [
                        "cell1",
                    ],
                    "name": "sys",
                    "used_by": [],
                },
                check_order=False,
            ),
        },
        {
            "cell_id": "cell1",
            "console": None,
            "op": "cell-op",
            "output": None,
            "run_id": IsUUID(),
            "serialization": None,
            "stale_inputs": None,
            "status": "queued",
            "timestamp": IsFloat(),
        },
        {
            "cell_id": "cell1",
            "op": "remove-ui-elements",
        },
        {
            "cell_id": "cell1",
            "console": [],
            "op": "cell-op",
            "output": None,
            "run_id": IsUUID(),
            "serialization": None,
            "stale_inputs": None,
            "status": "running",
            "timestamp": IsFloat(),
        },
        {
            "op": "variable-values",
            "variables": IsList(
                {
                    "datatype": "int",
                    "name": "x",
                    "value": "42",
                },
                {
                    "datatype": "module",
                    "name": "sys",
                    "value": "sys",
                },
                check_order=False,
            ),
        },
        {
            "cell_id": "cell1",
            "console": None,
            "op": "cell-op",
            "output": {
                "channel": "output",
                "data": "",
                "mimetype": "text/plain",
                "timestamp": IsFloat(),
            },
            "run_id": IsUUID(),
            "serialization": None,
            "stale_inputs": None,
            "status": None,
            "timestamp": IsFloat(),
        },
        {
            "cell_id": "cell1",
            "console": None,
            "op": "cell-op",
            "output": None,
            "run_id": None,
            "serialization": None,
            "stale_inputs": None,
            "status": "idle",
            "timestamp": IsFloat(),
        },
        {
            "op": "completed-run",
        },
        {
            "cell_id": "cell1",
            "console": {
                "channel": "stdout",
                "data": "stdout\n",
                "mimetype": "text/plain",
                "timestamp": IsFloat(),
            },
            "op": "cell-op",
            "output": None,
            "run_id": None,
            "serialization": None,
            "stale_inputs": None,
            "status": None,
            "timestamp": IsFloat(),
        },
        {
            "cell_id": "cell1",
            "console": {
                "channel": "stderr",
                "data": "stderr\n",
                "mimetype": "text/plain",
                "timestamp": IsFloat(),
            },
            "op": "cell-op",
            "output": None,
            "run_id": None,
            "serialization": None,
            "stale_inputs": None,
            "status": None,
            "timestamp": IsFloat(),
        },
    ]

    process.terminate()
    process.wait(timeout=2)
    queue_manager.close_queues()


@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")
def test_queue_manager_connection():
    """Test creating and connecting queue managers."""
    from marimo._ipc import QueueManager

    host_manager, connection_info = QueueManager.create()
    client_manager = QueueManager.connect(connection_info)

    test_request = ExecuteCellsCommand(
        cell_ids=[CellId_t("cell1")],
        codes=["print('test')"],
    )
    host_manager.control_queue.put(test_request)
    assert client_manager.control_queue.get(timeout=1) == test_request

    kernel_message = b'{"op": "test-op", "data": "test"}'
    client_manager.stream_queue.put(kernel_message)
    assert host_manager.stream_queue.get(timeout=1) == kernel_message

    host_manager.close_queues()
    client_manager.close_queues()


class TestMsgpackIPC:
    """Test msgspec msgpack serialization for IPC channels.

    Each IPC channel has a specific type that must survive encode/decode.
    These tests mirror the channel types in Connection.create/connect:

        control:         CommandMessage (discriminated union)
        ui_element:      BatchableCommand (discriminated union)
        completion:      CodeCompletionCommand
        input:           str
        win32_interrupt: bool
        stream:          bytes
    """

    def test_control_channel(self) -> None:
        """CommandMessage union dispatches to the correct subtype."""
        cmd = ExecuteCellsCommand(cell_ids=["c1"], codes=["x=1"])
        encoded = msgspec.msgpack.encode(cmd)
        decoded = msgspec.msgpack.Decoder(CommandMessage).decode(encoded)
        assert decoded == cmd

    def test_control_channel_with_http_request(self) -> None:
        """HTTPRequest (a @dataclass) survives the round-trip as a nested field."""
        req = HTTPRequest(
            url={"path": "/run"},
            base_url={"path": "/"},
            headers={"content-type": "application/json"},
            query_params={"key": ["val1", "val2"]},
            path_params={},
            cookies={},
            meta={},
            user={},
        )
        cmd = ExecuteCellsCommand(cell_ids=["c1"], codes=["x=1"], request=req)
        encoded = msgspec.msgpack.encode(cmd)
        decoded = msgspec.msgpack.Decoder(CommandMessage).decode(encoded)
        assert decoded.request is not None
        assert decoded.request["url"]["path"] == "/run"
        assert decoded.request["query_params"]["key"] == ["val1", "val2"]

    def test_ui_element_channel(self) -> None:
        """BatchableCommand union round-trips both member types."""
        ui = UpdateUIElementCommand(
            object_ids=["e1"], values=[{"nested": [1, 2]}], token="tok"
        )
        encoded = msgspec.msgpack.encode(ui)
        assert msgspec.msgpack.Decoder(BatchableCommand).decode(encoded) == ui

        msg = ModelUpdateMessage(state={"k": "v"}, buffer_paths=[])
        model = ModelCommand(
            model_id="m1", message=msg, buffers=[b"buf"], token="tok"
        )
        encoded = msgspec.msgpack.encode(model)
        assert (
            msgspec.msgpack.Decoder(BatchableCommand).decode(encoded) == model
        )

    def test_completion_channel(self) -> None:
        cmd = CodeCompletionCommand(id="t", document="x = 1", cell_id="c1")
        encoded = msgspec.msgpack.encode(cmd)
        decoded = msgspec.msgpack.Decoder(CodeCompletionCommand).decode(
            encoded
        )
        assert decoded == cmd

    def test_primitive_channels(self) -> None:
        """str (input), bool (win32_interrupt), and bytes (stream) channels."""
        for value, typ in [
            ("user_input", str),
            (True, bool),
            (b'{"op": "cell-op"}', bytes),
        ]:
            encoded = msgspec.msgpack.encode(value)
            assert msgspec.msgpack.Decoder(typ).decode(encoded) == value

    def test_unknown_fields_are_ignored(self) -> None:
        """Decoder silently drops fields it doesn't recognize.

        This matters when the sender is newer than the receiver (e.g.
        a field was added to a command). msgspec must not reject the
        message — it should decode the known fields and discard the rest.
        """

        # A "V2" struct with the same tag but an extra field
        class ExecuteCellsCommandV2(
            msgspec.Struct,
            rename="camel",
            tag_field="type",
            tag="execute-cells",
        ):
            cell_ids: list[str]
            codes: list[str]
            new_field: str = "added_later"

        v2 = ExecuteCellsCommandV2(
            cell_ids=["c1"], codes=["x=1"], new_field="extra"
        )
        encoded = msgspec.msgpack.encode(v2)

        decoded = msgspec.msgpack.Decoder(CommandMessage).decode(encoded)
        assert type(decoded) is ExecuteCellsCommand
        assert decoded.cell_ids == ["c1"]
        assert decoded.codes == ["x=1"]

    def test_missing_optional_fields_get_defaults(self) -> None:
        """Decoder fills in defaults for fields the sender didn't include.

        This matters when the receiver is newer than the sender (e.g.
        a field was added with a default, but the sender hasn't been
        updated yet).
        """

        class ExecuteCellsCommandV2(
            msgspec.Struct,
            rename="camel",
            tag_field="type",
            tag="execute-cells",
        ):
            cell_ids: list[str]
            codes: list[str]
            new_field: str = "default_value"

        old = ExecuteCellsCommand(cell_ids=["c1"], codes=["x=1"])
        encoded = msgspec.msgpack.encode(old)

        decoded = msgspec.msgpack.Decoder(ExecuteCellsCommandV2).decode(
            encoded
        )
        assert decoded.cell_ids == ["c1"]
        assert decoded.new_field == "default_value"