Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

17.0 KB
Raw Blame
# Copyright 2026 Marimo. All rights reserved.
"""Unit tests for the marimo-free `stream_transaction_install` engine.

micropip itself only runs inside Pyodide, so we monkeypatch its package-manager
singleton and `micropip.transaction.Transaction` with stubs that simulate the
resolution and install steps deterministically. These stubs prove the engine's
logic, not that it still matches micropip's private API.
"""

from __future__ import annotations

import asyncio
import sys
import types
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any

if TYPE_CHECKING:
    import pytest


@dataclass
class _FakeWheel:
    name: str
    install_error: Exception | None = None
    install_event: asyncio.Event | None = None

    async def install(self, _wheel_base: Any, _compat: Any) -> None:
        if self.install_event is not None:
            await self.install_event.wait()
        if self.install_error is not None:
            raise self.install_error


@dataclass
class _FakeTransaction:
    wheels: list[_FakeWheel] = field(default_factory=list)
    # micropip stores `Requirement`s here; strings cover older releases.
    failed: list[Any] = field(default_factory=list)
    pyodide_packages: list[tuple[str, str, str]] = field(default_factory=list)
    seen_requirements: list[str] = field(default_factory=list)
    init_kwargs: dict[str, Any] = field(default_factory=dict)

    async def gather_requirements(self, requirements: list[str]) -> None:
        self.seen_requirements.extend(requirements)


class _FakeCompatLayer:
    def __init__(self) -> None:
        self.loaded: list[list[str]] = []

    def to_js(self, value: Any) -> Any:
        return value

    async def loadPackage(self, names: list[str]) -> None:
        self.loaded.append(list(names))


@dataclass
class _FakeMicropipManager:
    compat_layer: _FakeCompatLayer = field(default_factory=_FakeCompatLayer)
    index_urls: list[str] = field(default_factory=lambda: ["https://default/"])
    constraints: list[str] = field(default_factory=list)


def _install_fake_micropip(
    monkeypatch: Any, fake_tx: _FakeTransaction
) -> _FakeMicropipManager:
    """Install fake `micropip` and `micropip.transaction` modules into
    sys.modules so the engine resolves to them.

    These stubs must track micropip's private API. A stub that drifts from the
    real thing hides the breakage it exists to catch.
    """
    fake_mgr = _FakeMicropipManager()

    micropip_mod = types.ModuleType("micropip")
    micropip_mod._package_manager_singleton = fake_mgr  # type: ignore[attr-defined]

    txn_mod = types.ModuleType("micropip.transaction")

    def _Transaction(**kwargs: Any) -> _FakeTransaction:
        fake_tx.init_kwargs = kwargs
        return fake_tx

    txn_mod.Transaction = _Transaction  # type: ignore[attr-defined]

    monkeypatch.setitem(sys.modules, "micropip", micropip_mod)
    monkeypatch.setitem(sys.modules, "micropip.transaction", txn_mod)
    return fake_mgr


async def _drain(agen: Any) -> list[tuple[str, bool]]:
    out: list[tuple[str, bool]] = []
    async for item in agen:
        out.append(item)
    return out


async def test_two_wheels_both_succeed(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    tx = _FakeTransaction(wheels=[_FakeWheel("foo"), _FakeWheel("bar")])
    _install_fake_micropip(monkeypatch, tx)

    results = await _drain(stream_transaction_install(["foo", "bar"]))
    assert sorted(results) == [("bar", True), ("foo", True)]


async def test_wheel_install_failure_no_double_yield(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    tx = _FakeTransaction(
        wheels=[
            _FakeWheel("foo", install_error=RuntimeError("boom")),
            _FakeWheel("bar"),
        ]
    )
    _install_fake_micropip(monkeypatch, tx)

    results = await _drain(stream_transaction_install(["foo", "bar"]))
    assert sorted(results) == [("bar", True), ("foo", False)]
    # Each requested package yields exactly once.
    assert [name for name, _ in results].count("foo") == 1


async def test_resolution_failure_yields_false(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """micropip records failures as `Requirement`s, not strings."""
    from packaging.requirements import Requirement

    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    tx = _FakeTransaction(failed=[Requirement("foo")])
    _install_fake_micropip(monkeypatch, tx)

    results = await _drain(stream_transaction_install(["foo"]))
    assert results == [("foo", False)]


async def test_failed_requirement_with_specifier_reports_base_name(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A versioned request must be reported back under the caller's spelling."""
    from packaging.requirements import Requirement

    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    tx = _FakeTransaction(failed=[Requirement("foo==1.2.3")])
    _install_fake_micropip(monkeypatch, tx)

    results = await _drain(stream_transaction_install(["foo==1.2.3"]))
    assert results == [("foo==1.2.3", False)]


async def test_pyodide_package_only_loadpackage_once(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    tx = _FakeTransaction(
        pyodide_packages=[("numpy", "1.26.0", "default")],
    )
    fake_mgr = _install_fake_micropip(monkeypatch, tx)

    results = await _drain(stream_transaction_install(["numpy"]))
    assert results == [("numpy", True)]
    assert fake_mgr.compat_layer.loaded == [["numpy"]]


async def test_index_urls_override_passed_to_transaction(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    tx = _FakeTransaction(wheels=[_FakeWheel("foo")])
    fake_mgr = _install_fake_micropip(monkeypatch, tx)

    override = ["https://my.private/simple", "https://extra/simple"]
    await _drain(stream_transaction_install(["foo"], index_urls=override))
    assert tx.init_kwargs["index_urls"] == override
    # And the singleton's was NOT used.
    assert tx.init_kwargs["index_urls"] is not fake_mgr.index_urls


async def test_index_urls_none_falls_back_to_singleton(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    tx = _FakeTransaction(wheels=[_FakeWheel("foo")])
    fake_mgr = _install_fake_micropip(monkeypatch, tx)

    await _drain(stream_transaction_install(["foo"], index_urls=None))
    assert tx.init_kwargs["index_urls"] is fake_mgr.index_urls


async def test_url_spec_passthrough(monkeypatch: pytest.MonkeyPatch) -> None:
    """PEP 508 URL specs in the packages list must reach gather_requirements
    unaltered — the engine shouldn't try to be clever about them."""
    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    tx = _FakeTransaction(wheels=[_FakeWheel("foo")])
    _install_fake_micropip(monkeypatch, tx)

    spec = "foo @ git+https://example.com/repo@deadbeef"
    await _drain(stream_transaction_install([spec]))
    assert spec in tx.seen_requirements


async def test_versions_appended(monkeypatch: pytest.MonkeyPatch) -> None:
    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    tx = _FakeTransaction(wheels=[_FakeWheel("foo")])
    _install_fake_micropip(monkeypatch, tx)

    await _drain(
        stream_transaction_install(["foo"], versions={"foo": "1.2.3"})
    )
    assert "foo==1.2.3" in tx.seen_requirements


async def test_streaming_order_first_done_first_yielded(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The engine should yield as each wheel completes, not in submission order."""
    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    fast_done = asyncio.Event()
    slow_done = asyncio.Event()
    tx = _FakeTransaction(
        wheels=[
            _FakeWheel("slow", install_event=slow_done),
            _FakeWheel("fast", install_event=fast_done),
        ]
    )
    _install_fake_micropip(monkeypatch, tx)

    async def driver() -> list[tuple[str, bool]]:
        results: list[tuple[str, bool]] = []
        agen = stream_transaction_install(["slow", "fast"])
        # Let `fast` finish first.
        fast_done.set()
        async for item in agen:
            results.append(item)
            if item[0] == "fast":
                # Now release `slow` so the generator can complete.
                slow_done.set()
        return results

    results = await driver()
    assert results[0] == ("fast", True)
    assert results[1] == ("slow", True)


async def test_versioned_spec_tracked_correctly(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A versioned spec ("foo==1.0") must be tracked by its base name so the
    wheel install ("foo") is recognized as fulfilling the request."""
    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    tx = _FakeTransaction(wheels=[_FakeWheel("foo")])
    _install_fake_micropip(monkeypatch, tx)

    results = await _drain(stream_transaction_install(["foo==1.0"]))
    # Yields the original spec string, not the bare name.
    assert results == [("foo==1.0", True)]


async def test_url_spec_tracked_correctly(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A PEP 508 URL spec ("foo @ git+…") must be tracked by base name too."""
    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    tx = _FakeTransaction(wheels=[_FakeWheel("foo")])
    _install_fake_micropip(monkeypatch, tx)

    spec = "foo @ git+https://example.com/repo@deadbeef"
    results = await _drain(stream_transaction_install([spec]))
    assert results == [(spec, True)]


async def test_reconciliation_uses_base_name_for_metadata_lookup(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A versioned spec that resolves transitively (no wheel/failed/pyodide
    entry of its own) is reconciled via `importlib.metadata`. The lookup must
    use the bare distribution name — `importlib.metadata.version("foo==1.0")`
    would always raise `PackageNotFoundError` and falsely report a failure."""
    import importlib.metadata as importlib_metadata

    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    # Empty transaction: nothing failed, no wheels, no pyodide packages, so the
    # spec falls through to the final importlib.metadata reconciliation pass.
    tx = _FakeTransaction()
    _install_fake_micropip(monkeypatch, tx)

    seen: list[str] = []

    def fake_version(name: str) -> str:
        seen.append(name)
        return "1.0"

    monkeypatch.setattr(importlib_metadata, "version", fake_version)

    results = await _drain(stream_transaction_install(["foo==1.0"]))
    # Reported back with the caller's original spelling, marked installed.
    assert results == [("foo==1.0", True)]
    # But looked up by the bare distribution name.
    assert seen == ["foo"]


async def _make_micropip_manager(monkeypatch: Any) -> Any:
    """Build a `MicropipPackageManager` outside Pyodide by stubbing the
    `is_pyodide()` guard that both the constructor path and `stream_install`
    assert on."""
    import marimo._runtime.packages.pypi_package_manager as ppm

    monkeypatch.setattr(ppm, "is_pyodide", lambda: True)
    return ppm.MicropipPackageManager()


async def test_manager_stream_install_happy_path(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The manager wraps the engine, marks attempted packages, and yields
    each package exactly once."""
    import marimo._runtime.packages.pypi_package_manager as ppm

    manager = await _make_micropip_manager(monkeypatch)

    async def fake_engine(packages: list[str], **_kwargs: Any) -> Any:
        for pkg in packages:
            yield (pkg, True)

    monkeypatch.setattr(ppm, "stream_transaction_install", fake_engine)

    results = await _drain(manager.stream_install(["foo", "bar"]))
    assert results == [("foo", True), ("bar", True)]
    assert manager._attempted_packages == {"foo", "bar"}
    # Each package yielded exactly once.
    names = [name for name, _ in results]
    assert sorted(names) == ["bar", "foo"]


async def test_manager_stream_install_fallback_no_reinstall(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """If the engine raises after yielding some packages, the manager falls
    back to the sequential base path for the *remaining* packages only —
    already-yielded packages are not re-installed, and every package is
    yielded exactly once."""
    import marimo._runtime.packages.pypi_package_manager as ppm

    manager = await _make_micropip_manager(monkeypatch)

    async def flaky_engine(packages: list[str], **_kwargs: Any) -> Any:
        del packages
        # First package resolves, then the private API "shifts".
        yield ("foo", True)
        raise AttributeError("Transaction internals moved")

    monkeypatch.setattr(ppm, "stream_transaction_install", flaky_engine)

    # Record which packages the fallback actually tries to install.
    installed: list[str] = []

    async def fake_install(package: str, *_args: Any, **_kwargs: Any) -> bool:
        installed.append(package)
        return True

    monkeypatch.setattr(manager, "_install", fake_install)

    results = await _drain(manager.stream_install(["foo", "bar", "baz"]))

    # `foo` came from the engine; `bar`/`baz` from the fallback.
    assert results == [("foo", True), ("bar", True), ("baz", True)]

    # The already-yielded `foo` was NOT re-installed by the fallback.
    assert installed == ["bar", "baz"]

    # Each requested package is yielded exactly once.
    names = [name for name, _ in results]
    assert sorted(names) == ["bar", "baz", "foo"]
    assert len(names) == len(set(names))


async def test_manager_stream_install_fallback_before_any_yield(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """If the engine raises before yielding anything, the fallback installs
    the full list."""
    import marimo._runtime.packages.pypi_package_manager as ppm

    manager = await _make_micropip_manager(monkeypatch)

    async def dead_engine(packages: list[str], **_kwargs: Any) -> Any:
        del packages
        raise ImportError("micropip.transaction unavailable")
        yield  # pragma: no cover - makes this an async generator

    monkeypatch.setattr(ppm, "stream_transaction_install", dead_engine)

    installed: list[str] = []

    async def fake_install(package: str, *_args: Any, **_kwargs: Any) -> bool:
        installed.append(package)
        return True

    monkeypatch.setattr(manager, "_install", fake_install)

    results = await _drain(manager.stream_install(["foo", "bar"]))
    assert results == [("foo", True), ("bar", True)]
    assert installed == ["foo", "bar"]


async def test_loadpackage_failure_yields_false_no_terminate(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A loadPackage exception must yield (name, False) per pyodide package
    rather than crashing the generator mid-stream."""
    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    class _BadCompatLayer(_FakeCompatLayer):
        async def loadPackage(self, names: list[str]) -> None:
            del names
            raise RuntimeError("pyodide load failed")

    fake_mgr = _FakeMicropipManager(compat_layer=_BadCompatLayer())
    tx = _FakeTransaction(
        wheels=[_FakeWheel("foo")],
        pyodide_packages=[("numpy", "1.26.0", "default")],
    )

    micropip_mod = types.ModuleType("micropip")
    micropip_mod._package_manager_singleton = fake_mgr  # type: ignore[attr-defined]
    txn_mod = types.ModuleType("micropip.transaction")

    def _Transaction(**kwargs: Any) -> _FakeTransaction:
        tx.init_kwargs = kwargs
        return tx

    txn_mod.Transaction = _Transaction  # type: ignore[attr-defined]
    monkeypatch.setitem(sys.modules, "micropip", micropip_mod)
    monkeypatch.setitem(sys.modules, "micropip.transaction", txn_mod)

    results = await _drain(stream_transaction_install(["foo", "numpy"]))
    # Generator completed (no crash); numpy yielded False from the failure.
    assert ("numpy", False) in results
    # foo wheel still succeeded independently.
    assert ("foo", True) in results


async def test_marker_environment_passed_to_transaction(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """micropip evaluates environment markers against this context."""
    from marimo._runtime.packages._micropip_streaming import (
        stream_transaction_install,
    )

    tx = _FakeTransaction(wheels=[_FakeWheel("foo")])
    _install_fake_micropip(monkeypatch, tx)

    await _drain(stream_transaction_install(["foo"]))
    assert tx.init_kwargs["ctx"]["python_version"] == ".".join(
        str(part) for part in sys.version_info[:2]
    )