Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat: build offline WASM exports from package mirrors (#11017)

Péter Gyarmati6472d2e02cb1
diff --git a/docs/guides/publishing/self_host_wasm.md b/docs/guides/publishing/self_host_wasm.md
index 689862e6db..071a81155e 100644
--- a/docs/guides/publishing/self_host_wasm.md
+++ b/docs/guides/publishing/self_host_wasm.md
@@ -23,7 +23,8 @@ python -m playwright install chromium
python -m marimo export html-wasm notebook.py -o dist --offline
```
-The export requires internet access. Playwright runs Pyodide to resolve
+The export requires internet access or
+[package mirrors](#using-package-mirrors). Playwright runs Pyodide to resolve
browser-compatible dependencies without executing notebook cells. The result
can be served from a local HTTP server or copied to a static host:
@@ -50,3 +51,38 @@ alternatives. Serve the export over HTTP even when using it offline.
Without `--offline`, the browser uses the default hosted Python runtime and
package sources.
+
+### Using package mirrors
+
+Pass `--pyodide-index-url` and `--pypi-index-url` to build the offline export
+from mirrors when the hosted sources are unreachable:
+
+```bash
+python -m marimo export html-wasm notebook.py -o dist --offline \
+ --pyodide-index-url https://mirror.example.com/pyodide/full/ \
+ --pypi-index-url https://mirror.example.com/pypi/simple/
+```
+
+`--pyodide-index-url` points at a mirror of the Pyodide `full/` distribution
+named in `marimo export html-wasm --help`. marimo downloads the runtime, its
+`pyodide-lock.json`, and the Pyodide packages the notebook needs from it, and
+stops the export when the mirror serves another Pyodide version. Imports resolve
+against that lockfile, so declare other imported packages, such as `anywidget`,
+in the notebook's inline script metadata.
+
+`--pypi-index-url` points at a package index, such as a PyPI mirror. marimo
+resolves each dependency from the Pyodide lockfile first and from this index
+otherwise, which also covers private packages hosted there. With
+`--pyodide-index-url`, the index provides `marimo-base` for your marimo version,
+`black`, and `sqlglot` for SQL notebooks. Alone, `--pypi-index-url` keeps
+marimo's hosted lockfile, which downloads marimo's own packages from
+`files.pythonhosted.org`, so pass both flags when that host is blocked. The index
+must allow anonymous downloads.
+
+marimo downloads from both mirrors in Python, so they need no CORS headers. The
+exported HTML loads only the bundled copies.
+
+The export runs [uv](https://docs.astral.sh/uv/getting-started/installation/),
+a Python package manager, to detect local modules, so install it on the machine
+that runs the export. Point uv at the same index, for example with
+[`UV_DEFAULT_INDEX`](https://docs.astral.sh/uv/reference/environment/#uv_default_index).
diff --git a/marimo/_cli/export/commands.py b/marimo/_cli/export/commands.py
index befca4a502..2c5a2d325a 100644
--- a/marimo/_cli/export/commands.py
+++ b/marimo/_cli/export/commands.py
@@ -8,6 +8,7 @@ from contextlib import contextmanager
from functools import partial
from pathlib import Path
from typing import TYPE_CHECKING, cast, get_args
+from urllib.parse import urlparse
import click
@@ -69,7 +70,10 @@ from marimo._export.requests import (
ScriptFileExportRequest,
WASMFileExportRequest,
)
-from marimo._pyodide.pyodide_constraints import PYODIDE_PYTHON_VERSION
+from marimo._pyodide.pyodide_constraints import (
+ PYODIDE_PYTHON_VERSION,
+ PYODIDE_VERSION,
+)
from marimo._schemas.export_options import (
ExportPDFPreset,
HTMLExportOptions,
@@ -81,6 +85,7 @@ from marimo._schemas.export_options import (
PDFRasterServer,
WASMExportOptions,
WASMMode,
+ WASMRuntimeConfig,
)
from marimo._server.utils import asyncio_run
from marimo._templates import get_default_asset_url
@@ -896,6 +901,33 @@ def pdf(
)
+def _source_url(
+ ctx: click.Context, param: click.Parameter, value: str | None
+) -> str | None:
+ del ctx, param
+ if value is None:
+ return None
+ not_http = (
+ "Expected an http(s) URL. Serve a local directory with "
+ "`python -m http.server`."
+ )
+ try:
+ url = urlparse(value)
+ except ValueError:
+ raise click.BadParameter(not_http) from None
+ if url.scheme not in ("http", "https") or not url.hostname:
+ raise click.BadParameter(not_http)
+ if url.username or url.password:
+ raise click.BadParameter(
+ "Credentials in the URL are not supported. Use a mirror that "
+ "allows anonymous downloads."
+ )
+ # Sources are base URLs that get file names and package paths appended.
+ if url.query or url.fragment:
+ raise click.BadParameter("Expected a URL without a query or fragment.")
+ return value
+
+
@click.command(
cls=ColoredCommand,
help="""Export a notebook as a WASM-powered standalone HTML file.
@@ -985,6 +1017,24 @@ assets from a CDN and open the HTML file directly. Internet access is required.
is_flag=True,
help="Download the Python runtime and notebook packages into the export directory.",
)
+ "--pyodide-index-url",
+ metavar="URL",
+ callback=_source_url,
+ help=(
+ f"Mirror of the Pyodide {PYODIDE_VERSION} full/ distribution to "
+ "bundle the runtime and its packages from. Requires --offline."
+ ),
+)
+ "--pypi-index-url",
+ metavar="URL",
+ callback=_source_url,
+ help=(
+ "Package index, such as a PyPI mirror, for packages outside the "
+ "Pyodide distribution. Requires --offline."
+ ),
+)
@click.argument(
"name",
required=True,
@@ -1003,6 +1053,8 @@ def html_wasm(
force: bool,
execute: bool,
offline: bool,
+ pyodide_index_url: str | None,
+ pypi_index_url: str | None,
args: tuple[str, ...],
) -> None:
"""Export a notebook as a WASM-powered standalone HTML file."""
@@ -1010,6 +1062,10 @@ def html_wasm(
raise click.UsageError(
"--single-file and --offline cannot be used together."
)
+ if not offline and (pyodide_index_url or pypi_index_url):
+ raise click.UsageError(
+ "--pyodide-index-url and --pypi-index-url require --offline."
+ )
if single_file and include_cloudflare:
raise click.UsageError(
"--single-file and --include-cloudflare cannot be used together."
@@ -1150,6 +1206,10 @@ def html_wasm(
mode=mode,
show_code=show_code,
asset_url=get_default_asset_url() if single_file else None,
+ runtime=WASMRuntimeConfig(
+ pyodide_index_url=pyodide_index_url,
+ pypi_index_url=pypi_index_url,
+ ),
)
if execute:
diff --git a/marimo/_export/offline.py b/marimo/_export/offline.py
index ab9f40a617..805b8e8f9c 100644
--- a/marimo/_export/offline.py
+++ b/marimo/_export/offline.py
@@ -10,7 +10,7 @@ import tempfile
from concurrent.futures import ThreadPoolExecutor
from contextlib import suppress
from pathlib import Path
-from typing import TypedDict
+from typing import TYPE_CHECKING, TypedDict
from urllib.parse import quote, unquote, urljoin, urlparse
from marimo._environments import script_metadata
@@ -21,6 +21,9 @@ from marimo._utils import requests
from marimo._utils.inline_script_metadata import PyProjectReader
from marimo._version import __version__
+if TYPE_CHECKING:
+ from playwright.async_api import Route # type: ignore[import-not-found]
+
class OfflineExportError(RuntimeError):
"""The runtime or package bundle could not be prepared."""
@@ -39,30 +42,41 @@ class _Lockfile(TypedDict):
packages: dict[str, _Package]
-_RESOLVE_PACKAGES = r"""async ({ indexURL, packageBaseUrl, lockfile, pypiIndexUrl, code, requirements }) => {
- const { loadPyodide } = await import(indexURL + "pyodide.mjs");
+_RESOLVE_PACKAGES = r"""async ({ indexURL, packageBaseUrl, lockfile, pypiIndexUrl, pyodideVersion, marimoVersion, code, requirements }) => {
+ const { loadPyodide, version } = await import(indexURL + "pyodide.mjs");
+ // The exported page loads this runtime with the frontend's Pyodide API.
+ if (version !== pyodideVersion) {
+ throw new Error(`${packageBaseUrl} serves Pyodide ${version}, but marimo needs Pyodide ${pyodideVersion}.`);
+ }
const pyodide = await loadPyodide({ indexURL, packageBaseUrl, lockFileContents: lockfile });
const errors = [];
const callbacks = { errorCallback: message => errors.push(message) };
await pyodide.loadPackage(
- ["micropip", "marimo-base", "docutils", "pygments", "jedi", "pyodide-http", "black"],
+ ["micropip", "packaging", "docutils", "pygments", "jedi", "pyodide-http"],
callbacks,
);
if (errors.length) throw new Error(errors.join("\n"));
pyodide.globals.set("requirements_json", JSON.stringify(requirements));
pyodide.globals.set("index_url", pypiIndexUrl);
pyodide.globals.set("base_url", location.href);
+ pyodide.globals.set("marimo_version", marimoVersion);
await pyodide.runPythonAsync(`
import json
import micropip
+import micropip.package_index
from packaging.requirements import Requirement
from packaging.utils import canonicalize_name
from urllib.parse import urljoin
if index_url:
micropip.set_index_urls(index_url)
+ # micropip checks pyodide-lock.json first only for its default index. Keep
+ # that order for mirrors so Pyodide-built pins such as pydantic-core hold.
+ micropip.package_index.DEFAULT_INDEX_URLS[:] = [index_url]
+# Stock Pyodide lockfiles lack marimo's packages, so micropip may use the index.
+await micropip.install([f"marimo-base=={marimo_version}", "black"])
+required = ["marimo-base", "black"]
requirements = []
-requirement_names = []
for value in json.loads(requirements_json):
requirement = Requirement(value)
if requirement.marker and not requirement.marker.evaluate():
@@ -72,26 +86,73 @@ for value in json.loads(requirements_json):
if requirement.url:
requirement.url = urljoin(base_url, requirement.url)
requirements.append(str(requirement))
- requirement_names.append(canonicalize_name(requirement.name))
+ required.append(canonicalize_name(requirement.name))
await micropip.install(requirements)
`);
// Keep the same SQL dependency policy as shouldLoadDuckDBPackages.
const usesDuckDB = /(^|\n)\s*(?:import\s+[^\n#]*\bduckdb\b|from\s+duckdb\b|[^\n#]*\bduckdb\s*\.)/.test(code);
- if (code.includes("mo.sql") || usesDuckDB || pyodide.runPython('"duckdb" in requirement_names')) {
- code += "\nimport duckdb, pandas, sqlglot";
+ if (code.includes("mo.sql") || usesDuckDB || pyodide.runPython('"duckdb" in required')) {
+ code += "\nimport duckdb, pandas";
if (code.includes("polars")) code += "\nimport pyarrow";
+ await pyodide.runPythonAsync('await micropip.install("sqlglot")\nrequired.append("sqlglot")');
}
await pyodide.loadPackagesFromImports(code, callbacks);
if (errors.length) throw new Error(errors.join("\n"));
- const frozen = JSON.parse(pyodide.runPython("micropip.freeze()"));
- const loaded = new Set([...Object.keys(pyodide.loadedPackages), "marimo-base"].map(name => name.toLowerCase().replace(/[-_.]+/g, "-")));
- frozen.packages = Object.fromEntries(Object.entries(frozen.packages).filter(([name]) => loaded.has(name)));
- return frozen;
+ return {
+ lockfile: JSON.parse(pyodide.runPython("micropip.freeze()")),
+ required: [...Object.keys(pyodide.loadedPackages), ...JSON.parse(pyodide.runPython("json.dumps(required)"))],
+ };
}"""
+class _Resolution(TypedDict):
+ lockfile: _Lockfile
+ required: list[str]
+
+
+def _lock_required(resolution: _Resolution) -> _Lockfile:
+ """Keep the lockfile entries in the dependency closure of `required`.
+
+ micropip only logs lockfile packages that fail to download, so the bundle
+ follows what the notebook needs rather than what loaded in the browser.
+ """
+ from packaging.utils import canonicalize_name
+
+ available = resolution["lockfile"]["packages"]
+ # Pyodide reports loaded packages by entry name, which can differ from the
+ # key: marimo's hosted lockfile names its marimo-base entry "marimo".
+ keys = {
+ canonicalize_name(package["name"]): key
+ for key, package in available.items()
+ }
+ packages: dict[str, _Package] = {}
+ pending = list(resolution["required"])
+ while pending:
+ name = canonicalize_name(pending.pop())
+ key = name if name in available else keys.get(name)
+ if key is None:
+ raise OfflineExportError(
+ f"No resolved package is named {name}. Check that direct "
+ "references use the wheel's project name."
+ )
+ if key in packages:
+ continue
+ packages[key] = available[key]
+ pending.extend(packages[key]["depends"])
+ return {"info": resolution["lockfile"]["info"], "packages": packages}
+
+
+def _fetch(url: str) -> bytes:
+ try:
+ return requests.get(url, timeout=60).raise_for_status().content
+ except (requests.RequestError, OSError) as error:
+ raise OfflineExportError(
+ f"Could not download {url}: {error}"
+ ) from error
+
+
def _download(url: str, target: Path, sha256: str | None = None) -> str:
- data = requests.get(url, timeout=60).raise_for_status().content
+ data = _fetch(url)
digest = hashlib.sha256(data).hexdigest()
if sha256 and digest != sha256:
raise ValueError(f"Checksum mismatch for {url}")
@@ -124,6 +185,32 @@ async def _download_all(
raise
+async def _fetch_in_python(route: Route) -> None:
+ """Fulfill a resolver page request from Python, so sources need no CORS."""
+ try:
+ # Without micropip's Accept, indexes answer with HTML. micropip 0.11
+ # leaves relative file URLs in JSON index pages unresolved.
+ response = await asyncio.to_thread(
+ requests.get, route.request.url, timeout=60
+ )
+ except Exception:
+ # An unsettled route stalls the page until the resolver timeout.
+ await route.abort()
+ return
+ headers = {key.lower(): value for key, value in response.headers.items()}
+ await route.fulfill(
+ status=response.status_code,
+ body=response.content,
+ headers={
+ "access-control-allow-origin": "*",
+ # micropip picks the index page parser by content type.
+ "content-type": headers.get(
+ "content-type", "application/octet-stream"
+ ),
+ },
+ )
+
+
def _rewrite_requirements(code: str, packages: dict[str, _Package]) -> str:
from packaging.requirements import Requirement
from packaging.specifiers import SpecifierSet
@@ -137,6 +224,10 @@ def _rewrite_requirements(code: str, packages: dict[str, _Package]) -> str:
requirement = Requirement(str(value))
name = canonicalize_name(requirement.name)
requirement.name = name
+ if name == "marimo":
+ # The exported runtime skips requirements named after loaded
+ # modules. With extras, it would ask micropip for marimo.
+ requirement.extras = set()
if requirement.url and name in packages:
requirement.url = None
requirement.specifier = SpecifierSet(
@@ -171,8 +262,12 @@ async def _resolve_packages(
browser = await playwright.chromium.launch()
try:
page = await browser.new_page()
+ origin = urljoin(page_url, "/")
+ await page.route(
+ lambda url: not url.startswith(origin), _fetch_in_python
+ )
await page.goto(page_url)
- return await asyncio.wait_for(
+ resolution = await asyncio.wait_for(
page.evaluate(
_RESOLVE_PACKAGES,
{
@@ -180,6 +275,8 @@ async def _resolve_packages(
"packageBaseUrl": package_base_url,
"lockfile": lockfile,
"pypiIndexUrl": pypi_index_url,
+ "pyodideVersion": PYODIDE_VERSION,
+ "marimoVersion": __version__,
"code": code,
"requirements": PyProjectReader.from_script(
code
@@ -190,6 +287,7 @@ async def _resolve_packages(
)
finally:
await browser.close()
+ return _lock_required(resolution)
def _publish_bundle(
@@ -197,10 +295,14 @@ def _publish_bundle(
output_dir: Path,
resolved: _Lockfile,
hashes: dict[Path, str],
+ pyodide_lockfile: _Lockfile,
) -> str:
from packaging.requirements import Requirement
from packaging.utils import canonicalize_name
+ pyodide_packages = {
+ canonicalize_name(name) for name in pyodide_lockfile["packages"]
+ }
for name, package in resolved["packages"].items():
name = canonicalize_name(Requirement(name).name)
filename = package["file_name"]
@@ -213,6 +315,11 @@ def _publish_bundle(
source.replace(target)
package["sha256"] = digest
package["file_name"] = f"../packages/{relative}"
+ if name in pyodide_packages:
+ # The exported runtime prefers index pages over the lockfile, and
+ # Pyodide-built wheels can require packages their entries omit,
+ # such as tzdata for pandas.
+ continue
index = staging / "packages" / "index" / name / "index.html"
index.parent.mkdir(parents=True, exist_ok=True)
index.write_text(
@@ -246,19 +353,29 @@ async def bundle_wasm_runtime(
local_wheel_paths: tuple[Path, ...] = (),
) -> tuple[str, WASMRuntimeConfig]:
"""Resolve packages in Pyodide and vendor them without executing cells."""
- index_url = sources.pyodide_index_url or (
- f"https://cdn.jsdelivr.net/pyodide/v{PYODIDE_VERSION}/full/"
- )
- index_url = index_url.rstrip("/") + "/"
- lockfile_url = sources.pyodide_lockfile_url or (
- f"https://wasm.marimo.app/pyodide-lock.json?v={__version__}"
- f"&pyodide=v{PYODIDE_VERSION}"
- )
- lockfile = await asyncio.to_thread(
- lambda: (
- requests.get(lockfile_url, timeout=60).raise_for_status().json()
+ if sources.pyodide_index_url:
+ index_url = sources.pyodide_index_url.rstrip("/") + "/"
+ # marimo's hosted lockfile only describes the default distribution.
+ default_lockfile_url = index_url + "pyodide-lock.json"
+ else:
+ index_url = (
+ f"https://cdn.jsdelivr.net/pyodide/v{PYODIDE_VERSION}/full/"
)
- )
+ default_lockfile_url = (
+ f"https://wasm.marimo.app/pyodide-lock.json?v={__version__}"
+ f"&pyodide=v{PYODIDE_VERSION}"
+ )
+ lockfile_url = sources.pyodide_lockfile_url or default_lockfile_url
+ try:
+ lockfile = json.loads(await asyncio.to_thread(_fetch, lockfile_url))
+ except json.JSONDecodeError as error:
+ raise OfflineExportError(
+ f"{lockfile_url} is not a Pyodide lockfile: {error}"
+ ) from error
+ pypi_index_url = sources.pypi_index_url
+ if pypi_index_url and "{package_name}" not in pypi_index_url:
+ # micropip appends "/<name>/", so pip-style trailing slashes would double.
+ pypi_index_url = pypi_index_url.rstrip("/")
await asyncio.to_thread(output_dir.mkdir, parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(
prefix=".marimo-offline-", dir=output_dir
@@ -292,7 +409,7 @@ async def bundle_wasm_runtime(
index_url=f"{server.base_url}/pyodide/",
package_base_url=index_url,
lockfile=lockfile,
- pypi_index_url=sources.pypi_index_url,
+ pypi_index_url=pypi_index_url,
)
downloads: dict[Path, tuple[str, str | None]] = {}
for package in resolved["packages"].values():
@@ -309,7 +426,9 @@ async def bundle_wasm_runtime(
downloads[target] = source
package["file_name"] = filename
hashes = await _download_all(downloads)
- lock_name = _publish_bundle(staging, output_dir, resolved, hashes)
+ lock_name = _publish_bundle(
+ staging, output_dir, resolved, hashes, lockfile
+ )
return _rewrite_requirements(
code, resolved["packages"]
), WASMRuntimeConfig(
diff --git a/tests/_cli/test_cli_export.py b/tests/_cli/test_cli_export.py
index 63f15c8526..8cf2fb8647 100644
--- a/tests/_cli/test_cli_export.py
+++ b/tests/_cli/test_cli_export.py
@@ -23,6 +23,7 @@ from marimo._dependencies.dependencies import DependencyManager
from marimo._export.local_modules import _ruff_import_graph
from marimo._export.local_wheels import _local_wheel_path
from marimo._output.utils import uri_decode_component
+from marimo._schemas.export_options import WASMRuntimeConfig
from marimo._session.state.serialize import get_session_cache_file
from marimo._utils import async_path
from marimo._utils.paths import marimo_package_path
@@ -287,9 +288,29 @@ class TestExportHTML:
assert mount_config["layout"] == {"type": "slides", "data": {}}
@staticmethod
- def test_cli_export_html_wasm_offline(tmp_path: Path) -> None:
- from marimo._schemas.export_options import WASMRuntimeConfig
-
+ @pytest.mark.parametrize(
+ ("source_args", "expected_sources"),
+ [
+ ((), WASMRuntimeConfig()),
+ (
+ (
+ "--pyodide-index-url",
+ "https://mirror.example/pyodide/",
+ "--pypi-index-url",
+ "https://mirror.example/simple/",
+ ),
+ WASMRuntimeConfig(
+ pyodide_index_url="https://mirror.example/pyodide/",
+ pypi_index_url="https://mirror.example/simple/",
+ ),
+ ),
+ ],
+ )
+ def test_cli_export_html_wasm_offline(
+ tmp_path: Path,
+ source_args: tuple[str, ...],
+ expected_sources: WASMRuntimeConfig,
+ ) -> None:
notebook = tmp_path / "notebook.py"
_write_minimal_wasm_notebook(notebook, ' "hello"\n return\n')
out_dir = tmp_path / "out"
@@ -307,7 +328,7 @@ class TestExportHTML:
local_wheel_paths: tuple[Path, ...],
):
assert output_dir == out_dir
- assert sources == WASMRuntimeConfig()
+ assert sources == expected_sources
assert local_wheel_paths == ()
return code, runtime
@@ -330,6 +351,7 @@ class TestExportHTML:
"--output",
str(out_dir),
"--offline",
+ *source_args,
)
_assert_success(result)
browser_check.assert_awaited_once_with()
@@ -339,6 +361,67 @@ class TestExportHTML:
assert 'data-pyodide-lockfile-url="./lockfile/test.json"' in html
assert 'data-pypi-index-url="./packages/index/"' in html
+ @staticmethod
+ @pytest.mark.parametrize(
+ "source_flag", ["--pyodide-index-url", "--pypi-index-url"]
+ )
+ def test_cli_export_html_wasm_sources_require_offline(
+ tmp_path: Path, source_flag: str
+ ) -> None:
+ notebook = tmp_path / "notebook.py"
+ _write_minimal_wasm_notebook(notebook, ' "hello"\n return\n')
+ out_dir = tmp_path / "out"
+ result = _run_export(
+ "html-wasm",
+ str(notebook),
+ "--output",
+ str(out_dir),
+ "--no-sandbox",
+ source_flag,
+ "https://mirror.example/",
+ )
+ assert result.exit_code == 2
+ assert "require --offline" in result.output
+ assert not out_dir.exists()
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("url", "message"),
+ [
+ ("./pyodide/", "Expected an http(s) URL"),
+ ("file:///tmp/pyodide/", "Expected an http(s) URL"),
+ ("http://[::1/simple/", "Expected an http(s) URL"),
+ (
+ "https://user:[email protected]/simple/",
+ "Credentials in the URL are not supported",
+ ),
+ (
+ "https://mirror.example/simple/?token=secret-token",
+ "Expected a URL without a query or fragment",
+ ),
+ ],
+ )
+ def test_cli_export_html_wasm_rejects_invalid_source_urls(
+ tmp_path: Path, url: str, message: str
+ ) -> None:
+ notebook = tmp_path / "notebook.py"
+ _write_minimal_wasm_notebook(notebook, ' "hello"\n return\n')
+ out_dir = tmp_path / "out"
+ result = _run_export(
+ "html-wasm",
+ str(notebook),
+ "--output",
+ str(out_dir),
+ "--offline",
+ "--no-sandbox",
+ "--pypi-index-url",
+ url,
+ )
+ assert result.exit_code == 2
+ assert message in result.output
+ assert "secret-token" not in result.output
+ assert not out_dir.exists()
+
@staticmethod
@pytest.mark.parametrize("playwright_installed", [False, True])
def test_cli_offline_requires_browser_before_export(
diff --git a/tests/_export/test_offline.py b/tests/_export/test_offline.py
index c94ebc41c6..823a444c5a 100644
--- a/tests/_export/test_offline.py
+++ b/tests/_export/test_offline.py
@@ -4,13 +4,23 @@ from __future__ import annotations
import asyncio
import hashlib
import json
+import re
import threading
from unittest import mock
+from urllib.parse import urlparse
import pytest
-from marimo._export.offline import _resolve_packages, bundle_wasm_runtime
+from marimo._export.offline import (
+ OfflineExportError,
+ _fetch_in_python,
+ _lock_required,
+ _resolve_packages,
+ bundle_wasm_runtime,
+)
+from marimo._pyodide.pyodide_constraints import PYODIDE_VERSION
from marimo._schemas.export_options import WASMRuntimeConfig
+from marimo._utils import requests
from marimo._utils.requests import Response
@@ -36,11 +46,12 @@ def resolver():
"install_dir": "site",
}
lock = {"info": {"python": "3.14.0"}, "packages": {"example": package}}
+ pyodide_lock = {"info": lock["info"], "packages": {}}
def fetch(url: str, **kwargs: object):
del kwargs
data = (
- json.dumps(lock).encode() if url.endswith("lock.json") else wheel
+ json.dumps(pyodide_lock).encode() if "lock.json" in url else wheel
)
return Response(200, data, {})
@@ -59,7 +70,7 @@ def resolver():
async def test_bundle_is_relocatable_and_preserves_source_config(
tmp_path, sources, resolver
):
- code = '# /// script\n# dependencies = ["Example @ https://wheels.example/example-1.0-py3-none-any.whl", "Marimo==0.24.2"]\n# ///\nraise RuntimeError("must not execute")\n'
+ code = '# /// script\n# dependencies = ["Example @ https://wheels.example/example-1.0-py3-none-any.whl", "Marimo[sql]==0.24.2"]\n# ///\nraise RuntimeError("must not execute")\n'
rewritten, runtime = await bundle_wasm_runtime(
code, tmp_path, sources=sources
)
@@ -71,7 +82,8 @@ async def test_bundle_is_relocatable_and_preserves_source_config(
== sources.pyodide_index_url
)
assert (
- resolver.call_args.kwargs["pypi_index_url"] == sources.pypi_index_url
+ resolver.call_args.kwargs["pypi_index_url"]
+ == "https://packages.example/simple"
)
assert runtime.pyodide_index_url == "./pyodide/"
assert runtime.pypi_index_url == "./packages/index/"
@@ -100,6 +112,180 @@ async def test_bundle_is_relocatable_and_preserves_source_config(
assert not list(tmp_path.glob(".marimo-offline-*"))
+async def test_pyodide_packages_install_from_the_bundled_lockfile(
+ tmp_path, sources, resolver
+):
+ # The exported runtime prefers index pages over the lockfile, and Pyodide's
+ # pandas wheel requires tzdata, which its lockfile entry omits.
+ packages = resolver.return_value["packages"]
+ packages["pandas"] = {
+ **packages["example"],
+ "name": "pandas",
+ "file_name": "pandas-3.0.2-cp314-cp314-pyodide_2026_0_wasm32.whl",
+ }
+ pyodide_lock = {"info": {}, "packages": {"pandas": packages["pandas"]}}
+ responses = {"lock.json": json.dumps(pyodide_lock).encode()}
+
+ def fetch(url: str, **kwargs: object):
+ del kwargs
+ data = responses.get(url.rsplit("/", 1)[-1], b"downloaded wheel")
+ return Response(200, data, {})
+
+ with mock.patch.object(requests, "get", side_effect=fetch):
+ _, runtime = await bundle_wasm_runtime(
+ "pass", tmp_path, sources=sources
+ )
+ assert runtime.pyodide_lockfile_url is not None
+ lock = json.loads((tmp_path / runtime.pyodide_lockfile_url).read_text())
+ assert sorted(lock["packages"]) == ["example", "pandas"]
+ index = tmp_path / "packages/index"
+ assert [path.name for path in index.iterdir()] == ["example"]
+
+
+async def test_custom_pyodide_distribution_supplies_runtime_and_lockfile(
+ tmp_path, resolver
+):
+ # Records the URLs requested from the resolver fixture's fake.
+ with mock.patch.object(requests, "get", wraps=requests.get) as get:
+ await bundle_wasm_runtime(
+ "pass",
+ tmp_path,
+ sources=WASMRuntimeConfig(
+ pyodide_index_url="https://mirror.example/pyodide"
+ ),
+ )
+ assert sorted(call.args[0] for call in get.call_args_list) == [
+ "https://mirror.example/pyodide/pyodide-lock.json",
+ "https://mirror.example/pyodide/pyodide.asm.mjs",
+ "https://mirror.example/pyodide/pyodide.asm.wasm",
+ "https://mirror.example/pyodide/pyodide.mjs",
+ "https://mirror.example/pyodide/python_stdlib.zip",
+ "https://wheels.example/example-1.0-py3-none-any.whl",
+ ]
+ assert (
+ resolver.call_args.kwargs["package_base_url"]
+ == "https://mirror.example/pyodide/"
+ )
+
+
+async def test_default_sources_are_the_hosted_distribution(tmp_path, resolver):
+ with mock.patch.object(requests, "get", wraps=requests.get) as get:
+ await bundle_wasm_runtime(
+ "pass", tmp_path, sources=WASMRuntimeConfig()
+ )
+ assert {
+ urlparse(call.args[0]).hostname for call in get.call_args_list
+ } == {
+ "cdn.jsdelivr.net",
+ "wasm.marimo.app",
+ "wheels.example",
+ }
+ assert resolver.call_args.kwargs["pypi_index_url"] is None
+
+
+ ("response", "message"),
+ [
+ (
+ Response(404, b"missing", {}),
+ "Could not download https://runtime.example/lock.json",
+ ),
+ (
+ Response(200, b"<!doctype html>", {}),
+ "https://runtime.example/lock.json is not a Pyodide lockfile",
+ ),
+ ],
+)
+async def test_lockfile_failures_name_the_source_url(
+ tmp_path, sources, resolver, response, message
+):
+ with (
+ mock.patch.object(requests, "get", return_value=response),
+ pytest.raises(OfflineExportError, match=re.escape(message)),
+ ):
+ await bundle_wasm_runtime("pass", tmp_path, sources=sources)
+ resolver.assert_not_called()
+
+
+def _package(name: str, *depends: str) -> dict[str, object]:
+ return {
+ "name": name,
+ "version": "1.0",
+ "file_name": f"{name}-1.0-py3-none-any.whl",
+ "sha256": None,
+ "depends": list(depends),
+ }
+
+
+def test_bundle_keeps_the_dependency_closure_of_required_packages():
+ lockfile = {
+ "info": {"python": "3.14.0"},
+ "packages": {
+ "marimo-base": _package("marimo", "msgspec"),
+ "msgspec": _package("msgspec"),
+ "numpy": _package("numpy"),
+ },
+ }
+ # Pyodide reports hosted marimo-base under its lockfile name, marimo.
+ assert _lock_required(
+ {"lockfile": lockfile, "required": ["marimo", "Marimo_Base"]}
+ ) == {
+ "info": {"python": "3.14.0"},
+ "packages": {
+ "marimo-base": _package("marimo", "msgspec"),
+ "msgspec": _package("msgspec"),
+ },
+ }
+
+
+def test_bundle_rejects_packages_missing_from_the_resolution():
+ lockfile = {"info": {}, "packages": {"black": _package("black", "click")}}
+ with pytest.raises(
+ OfflineExportError, match="No resolved package is named click"
+ ):
+ _lock_required({"lockfile": lockfile, "required": ["black"]})
+
+
+async def test_page_requests_are_fetched_in_python_with_cors():
+ route = mock.AsyncMock()
+ route.request.url = "https://mirror.example/simple/humanize/"
+ route.request.headers = {"accept": "application/vnd.pypi.simple.v1+json"}
+ page = Response(200, b"<a>", {"Content-Type": "text/html"})
+ with mock.patch.object(requests, "get", return_value=page) as get:
+ await _fetch_in_python(route)
+ # Devpi serves JSON with relative file URLs when asked, which micropip
+ # cannot resolve. Its HTML pages resolve against the page URL.
+ get.assert_called_once_with(
+ "https://mirror.example/simple/humanize/", timeout=60
+ )
+ route.fulfill.assert_awaited_once_with(
+ status=200,
+ body=b"<a>",
+ headers={
+ "access-control-allow-origin": "*",
+ "content-type": "text/html",
+ },
+ )
+
+
+async def test_failed_page_requests_are_aborted():
+ route = mock.AsyncMock()
+ route.request.url = "https://mirror.example/simple/humanize/"
+ route.request.headers = {}
+ with mock.patch.object(
+ requests, "get", side_effect=ConnectionResetError("reset")
+ ):
+ await _fetch_in_python(route)
+ route.abort.assert_awaited_once_with()
+ route.fulfill.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_bundle_rejects_package_names_outside_the_index(
tmp_path, sources, resolver
@@ -199,9 +385,10 @@ async def test_cancelled_bundle_joins_downloads_before_cleanup(
assert list(tmp_path.iterdir()) == []
[email protected]("phase", ["bootstrap", "imports"])
-async def test_resolver_rejects_reported_package_failures(tmp_path, phase):
+async def _resolve_with_stub(
+ tmp_path, *, version=PYODIDE_VERSION, load_package="", load_imports=""
+):
+ """Run the resolver against a stub `pyodide.mjs` served next to the page."""
pytest.importorskip("playwright.async_api")
from marimo._export._html_asset_server import HtmlAssetServer
from marimo._export.dependencies import _is_playwright_chromium_installed
@@ -209,30 +396,86 @@ async def test_resolver_rejects_reported_package_failures(tmp_path, phase):
if not await _is_playwright_chromium_installed():
pytest.skip("Chromium not installed")
(tmp_path / "pyodide.mjs").write_text(
+ f"export const version = {json.dumps(version)};"
"export async function loadPyodide() { return {"
- "globals: {set() {}}, runPythonAsync: async () => {}, runPython: () => false,"
- "loadPackage: async (_, callbacks) => {"
- + (
- 'callbacks.errorCallback("bootstrap download failed");'
- if phase == "bootstrap"
- else ""
- )
- + "}, loadPackagesFromImports: async (_, callbacks) => {"
- + (
- 'callbacks.errorCallback("imports download failed");'
- if phase == "imports"
- else ""
- )
- + "}};}"
+ "globals: {set() {}}, loadedPackages: {}, runPythonAsync: async () => {},"
+ "runPython: code => ({'micropip.freeze()': '{\"info\": {}, \"packages\": {}}',"
+ "'json.dumps(required)': '[]'})[code] ?? false,"
+ f"loadPackage: async (_, callbacks) => {{ {load_package} }},"
+ f"loadPackagesFromImports: async (_, callbacks) => {{ {load_imports} }},"
+ "};}"
)
with HtmlAssetServer(directory=tmp_path, route="/resolve.html") as server:
server.set_html("<!doctype html>")
- with pytest.raises(Exception, match=f"{phase} download failed"):
- await _resolve_packages(
+ return await asyncio.wait_for(
+ _resolve_packages(
"import numpy",
page_url=server.page_url,
index_url=server.base_url + "/",
package_base_url=server.base_url + "/",
lockfile={"info": {}, "packages": {}},
pypi_index_url=None,
+ ),
+ timeout=60,
+ )
+
+
+def _load_from(url: str) -> str:
+ return (
+ f"const response = await fetch({json.dumps(url)}).catch(() => null);"
+ 'if (!response?.ok) callbacks.errorCallback("mirror unavailable");'
+ )
+
+
[email protected]("phase", ["bootstrap", "imports"])
+async def test_resolver_rejects_reported_package_failures(tmp_path, phase):
+ failure = f'callbacks.errorCallback("{phase} download failed");'
+ with pytest.raises(Exception, match=f"{phase} download failed"):
+ await _resolve_with_stub(
+ tmp_path,
+ load_package=failure if phase == "bootstrap" else "",
+ load_imports=failure if phase == "imports" else "",
+ )
+
+
+async def test_resolver_rejects_other_pyodide_versions(tmp_path):
+ expected = (
+ f"serves Pyodide 0.0.0, but marimo needs Pyodide {PYODIDE_VERSION}"
+ )
+ with pytest.raises(Exception, match=re.escape(expected)):
+ await _resolve_with_stub(tmp_path, version="0.0.0")
+
+
+async def test_resolver_loads_packages_from_mirrors_without_cors(tmp_path):
+ from marimo._export._html_asset_server import HtmlAssetServer
+
+ mirror = tmp_path / "mirror"
+ mirror.mkdir()
+ (mirror / "package.whl").write_bytes(b"wheel")
+ # SimpleHTTPRequestHandler sends no Access-Control-Allow-Origin header.
+ with HtmlAssetServer(directory=mirror, route="/unused") as server:
+ resolved = await _resolve_with_stub(
+ tmp_path, load_package=_load_from(f"{server.base_url}/package.whl")
+ )
+ assert resolved == {"info": {}, "packages": {}}
+
+
+async def test_resolver_fails_fast_when_mirror_connections_fail(tmp_path):
+ # Holding the port avoids reuse races, and closing each connection fails
+ # the request at once on every platform.
+ mirror = await asyncio.start_server(
+ lambda _reader, writer: writer.close(), "127.0.0.1", 0
+ )
+ port = mirror.sockets[0].getsockname()[1]
+ async with mirror:
+ with pytest.raises(Exception, match="mirror unavailable"):
+ await _resolve_with_stub(
+ tmp_path,
+ load_package=_load_from(
+ f"http://127.0.0.1:{port}/package.whl"
+ ),
)