Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

18.6 KB
Raw Blame
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations

from typing import Any
from unittest.mock import MagicMock, patch

import pytest
from inline_snapshot import snapshot

from marimo._data._external_storage.huggingface import (
    HuggingfaceApi,
    _hub_path_for_repo,
    _parse_hub_path,
    _ResolvedHubPath,
)
from marimo._data._external_storage.models import (
    StorageEntry,
    StorageListResult,
)
from marimo._dependencies.dependencies import DependencyManager
from marimo._types.ids import VariableName

HAS_HF = DependencyManager.huggingface_hub.has()


class TestParseHubPath:
    def test_root(self) -> None:
        assert _parse_hub_path("") == snapshot(_ResolvedHubPath(kind="root"))
        assert _parse_hub_path("   ") == snapshot(
            _ResolvedHubPath(kind="root")
        )

    def test_dataset_repo(self) -> None:
        resolved = _parse_hub_path("datasets/scikit-learn/Fish")
        assert resolved == snapshot(
            _ResolvedHubPath(
                kind="repo",
                repo_type="dataset",
                repo_id="scikit-learn/Fish",
            )
        )

    def test_dataset_nested(self) -> None:
        resolved = _parse_hub_path("datasets/scikit-learn/Fish/data/train.csv")
        assert resolved == snapshot(
            _ResolvedHubPath(
                kind="repo",
                repo_type="dataset",
                repo_id="scikit-learn/Fish",
                path_in_repo="data/train.csv",
            )
        )

    def test_model_repo(self) -> None:
        resolved = _parse_hub_path("google/bert-base-uncased")
        assert resolved == snapshot(
            _ResolvedHubPath(
                kind="repo",
                repo_type="model",
                repo_id="google/bert-base-uncased",
            )
        )

    def test_model_nested(self) -> None:
        resolved = _parse_hub_path("google/bert-base-uncased/weights.bin")
        assert resolved == snapshot(
            _ResolvedHubPath(
                kind="repo",
                repo_type="model",
                repo_id="google/bert-base-uncased",
                path_in_repo="weights.bin",
            )
        )

    def test_bucket(self) -> None:
        # Bucket ids are namespaced like repo ids ("namespace/bucket-name").
        resolved = _parse_hub_path(
            "buckets/my-org/my-bucket/data/file.parquet"
        )
        assert resolved == snapshot(
            _ResolvedHubPath(
                kind="bucket",
                bucket_id="my-org/my-bucket",
                path_in_repo="data/file.parquet",
            )
        )

    def test_bucket_repo_root(self) -> None:
        resolved = _parse_hub_path("buckets/my-org/my-bucket")
        assert resolved == snapshot(
            _ResolvedHubPath(kind="bucket", bucket_id="my-org/my-bucket")
        )

    def test_bucket_incomplete(self) -> None:
        with pytest.raises(
            ValueError, match="Incomplete Hugging Face Hub path"
        ):
            _parse_hub_path("buckets/my-bucket")

    def test_ignores_empty_segments_from_double_slashes(self) -> None:
        # A double slash shouldn't leak an empty segment into repo_id/
        # namespaced_id, matching the frontend's `.filter(Boolean)`.
        assert _parse_hub_path("datasets//scikit-learn/Fish") == snapshot(
            _ResolvedHubPath(
                kind="repo",
                repo_type="dataset",
                repo_id="scikit-learn/Fish",
            )
        )
        assert _parse_hub_path("google//bert-base-uncased") == snapshot(
            _ResolvedHubPath(
                kind="repo",
                repo_type="model",
                repo_id="google/bert-base-uncased",
            )
        )
        assert _parse_hub_path(
            "google/bert-base-uncased//weights.bin"
        ) == snapshot(
            _ResolvedHubPath(
                kind="repo",
                repo_type="model",
                repo_id="google/bert-base-uncased",
                path_in_repo="weights.bin",
            )
        )


class TestHubPathForRepo:
    def test_paths(self) -> None:
        assert _hub_path_for_repo("dataset", "org/name") == "datasets/org/name"
        assert _hub_path_for_repo("space", "org/name") == "spaces/org/name"
        assert _hub_path_for_repo("model", "org/name") == "org/name"


@pytest.mark.skipif(not HAS_HF, reason="huggingface_hub not installed")
class TestHuggingfaceApi:
    def _make_backend(self, api: Any, name: str = "hf") -> HuggingfaceApi:
        return HuggingfaceApi(api, VariableName(name))

    def test_list_repo_entries(self) -> None:
        from huggingface_hub import RepoFile

        mock_api = MagicMock()
        mock_file = MagicMock(spec=RepoFile)
        mock_file.path = "Fish.csv"
        mock_file.size = 5862
        mock_file.last_commit = None
        mock_api.list_repo_tree.return_value = [mock_file]

        backend = self._make_backend(mock_api)
        result = backend.list_entries(prefix="datasets/scikit-learn/Fish")

        mock_api.list_repo_tree.assert_called_once_with(
            "scikit-learn/Fish",
            path_in_repo=None,
            repo_type="dataset",
            recursive=False,
        )
        assert result.entries == snapshot(
            [
                StorageEntry(
                    path="datasets/scikit-learn/Fish/Fish.csv",
                    kind="file",
                    size=5862,
                    last_modified=None,
                    metadata={},
                    mime_type="text/csv",
                )
            ]
        )

    def test_list_bucket_entries(self) -> None:
        from huggingface_hub import BucketFile

        mock_api = MagicMock()
        mock_file = MagicMock(spec=BucketFile)
        mock_file.path = "data/file.parquet"
        mock_file.size = 1024
        mock_file.uploaded_at = None
        mock_api.list_bucket_tree.return_value = [mock_file]

        backend = self._make_backend(mock_api)
        result = backend.list_entries(prefix="buckets/my-org/my-bucket")

        # Bucket ids are namespaced ("namespace/bucket-name"), not a single
        # path segment, so the full namespace must be forwarded to the API.
        mock_api.list_bucket_tree.assert_called_once_with(
            "my-org/my-bucket",
            prefix=None,
            recursive=False,
        )
        assert result.entries == snapshot(
            [
                StorageEntry(
                    path="buckets/my-org/my-bucket/data/file.parquet",
                    kind="file",
                    size=1024,
                    last_modified=None,
                    metadata={},
                    mime_type=None,
                )
            ]
        )

    @pytest.mark.asyncio
    async def test_get_bucket_entry_file(self) -> None:
        from huggingface_hub import BucketFile

        mock_api = MagicMock()
        mock_file = MagicMock(spec=BucketFile)
        mock_file.path = "data/file.parquet"
        mock_file.size = 1024
        mock_file.uploaded_at = None
        mock_api.get_bucket_paths_info.return_value = [mock_file]

        backend = self._make_backend(mock_api)
        entry = await backend.get_entry(
            "buckets/my-org/my-bucket/data/file.parquet"
        )

        # A targeted lookup, not a full directory listing.
        mock_api.get_bucket_paths_info.assert_called_once_with(
            "my-org/my-bucket", ["data/file.parquet"]
        )
        mock_api.list_bucket_tree.assert_not_called()
        assert entry == snapshot(
            StorageEntry(
                path="buckets/my-org/my-bucket/data/file.parquet",
                kind="file",
                size=1024,
                last_modified=None,
                metadata={},
                mime_type=None,
            )
        )

    @pytest.mark.asyncio
    async def test_get_bucket_entry_directory_falls_back_to_listing(
        self,
    ) -> None:
        # get_bucket_paths_info only resolves files (nonexistent paths are
        # silently ignored), so directories must fall back to a listing.
        from huggingface_hub import BucketFolder

        mock_api = MagicMock()
        mock_api.get_bucket_paths_info.return_value = []
        mock_folder = MagicMock(spec=BucketFolder)
        mock_folder.path = "data"
        mock_folder.uploaded_at = None
        mock_api.list_bucket_tree.return_value = [mock_folder]

        backend = self._make_backend(mock_api)
        entry = await backend.get_entry("buckets/my-org/my-bucket/data")

        assert entry == snapshot(
            StorageEntry(
                path="buckets/my-org/my-bucket/data",
                kind="directory",
                size=0,
                last_modified=None,
                metadata={},
                mime_type=None,
            )
        )

    @pytest.mark.asyncio
    async def test_get_bucket_entry_not_found(self) -> None:
        mock_api = MagicMock()
        mock_api.get_bucket_paths_info.return_value = []
        mock_api.list_bucket_tree.return_value = []

        backend = self._make_backend(mock_api)
        with pytest.raises(ValueError, match="not found"):
            await backend.get_entry("buckets/my-org/my-bucket/missing.txt")

    def test_list_root_entries_with_author(self) -> None:
        mock_api = MagicMock()
        mock_api.whoami.return_value = {"name": "marimo-team"}
        mock_dataset = MagicMock()
        mock_dataset.id = "marimo-team/demo"
        mock_api.list_datasets.return_value = [mock_dataset]
        mock_api.list_models.return_value = []
        mock_api.list_spaces.return_value = []
        mock_api.list_buckets.return_value = []

        backend = self._make_backend(mock_api)
        result = backend.list_entries(prefix="")

        mock_api.list_datasets.assert_called_once_with(
            author="marimo-team", limit=50
        )
        assert result.entries == snapshot(
            [
                StorageEntry(
                    path="datasets/marimo-team/demo",
                    kind="directory",
                    size=0,
                    last_modified=None,
                    metadata={},
                    mime_type=None,
                )
            ]
        )

    def test_list_root_entries_with_non_string_username(self) -> None:
        # If whoami() ever returns a non-string name (e.g. None), we must
        # not filter listings by the literal string "None".
        mock_api = MagicMock()
        mock_api.whoami.return_value = {"name": None}
        mock_api.list_datasets.return_value = []
        mock_api.list_models.return_value = []
        mock_api.list_spaces.return_value = []
        mock_api.list_buckets.return_value = []

        backend = self._make_backend(mock_api)
        backend.list_entries(prefix="")

        mock_api.list_datasets.assert_called_once_with(author=None, limit=50)

    def test_list_root_entries_tolerates_partial_failures(self) -> None:
        # A transient failure in one listing call (e.g. rate limiting)
        # shouldn't prevent the others from rendering.
        mock_api = MagicMock()
        mock_api.whoami.return_value = {"name": "marimo-team"}
        mock_api.list_datasets.side_effect = Exception("rate limited")
        mock_model = MagicMock()
        mock_model.id = "marimo-team/model"
        mock_api.list_models.return_value = [mock_model]
        mock_api.list_spaces.side_effect = Exception("rate limited")
        mock_api.list_buckets.return_value = []

        backend = self._make_backend(mock_api)
        result = backend.list_entries(prefix="")

        assert result.entries == snapshot(
            [
                StorageEntry(
                    path="marimo-team/model",
                    kind="directory",
                    size=0,
                    last_modified=None,
                    metadata={},
                    mime_type=None,
                )
            ]
        )

    def test_protocol_and_backend_type(self) -> None:
        backend = self._make_backend(MagicMock())
        assert backend.protocol == "hf"
        assert backend.backend_type == "huggingface"
        assert backend.display_name == "Hugging Face Hub"
        assert backend.root_path is None

    def test_is_compatible(self) -> None:
        from huggingface_hub import HfApi

        assert HuggingfaceApi.is_compatible(HfApi()) is True
        assert HuggingfaceApi.is_compatible("not hf") is False

    @pytest.mark.asyncio
    async def test_sign_download_url(self) -> None:
        mock_api = MagicMock()
        mock_api.endpoint = "https://huggingface.co"
        backend = self._make_backend(mock_api)
        url = await backend.sign_download_url(
            "datasets/scikit-learn/Fish/Fish.csv"
        )
        assert url is not None
        assert "huggingface.co" in url
        assert "Fish.csv" in url

    @pytest.mark.asyncio
    async def test_download_repo_file(self) -> None:
        mock_api = MagicMock()
        mock_api.token = None
        mock_api.endpoint = "https://huggingface.co"
        backend = self._make_backend(mock_api)

        with patch(
            "huggingface_hub.hf_hub_download",
            return_value="/tmp/Fish.csv",
        ) as mock_download:
            with patch(
                "pathlib.Path.read_bytes",
                return_value=b"csv-data",
            ):
                data = await backend.download(
                    "datasets/scikit-learn/Fish/Fish.csv"
                )

        mock_download.assert_called_once_with(
            repo_id="scikit-learn/Fish",
            filename="Fish.csv",
            repo_type="dataset",
            token=None,
            endpoint="https://huggingface.co",
        )
        assert data == b"csv-data"

    @pytest.mark.asyncio
    async def test_download_bucket_file(self) -> None:
        mock_api = MagicMock()
        mock_api.token = None
        backend = self._make_backend(mock_api)

        def _fake_download_bucket_files(
            _bucket_id: str, files: list[tuple[str, str]], **_kwargs: Any
        ) -> None:
            _, tmp_path = files[0]
            with open(tmp_path, "wb") as f:
                f.write(b"bucket-data")

        mock_api.download_bucket_files.side_effect = (
            _fake_download_bucket_files
        )

        data = await backend.download(
            "buckets/my-org/my-bucket/data/file.parquet"
        )

        # Bucket ids are namespaced ("namespace/bucket-name"), not a single
        # path segment, so the full namespace must be forwarded to the API.
        args, kwargs = mock_api.download_bucket_files.call_args
        assert args[0] == "my-org/my-bucket"
        assert args[1][0][0] == "data/file.parquet"
        assert kwargs["token"] is None
        assert data == b"bucket-data"

    @pytest.mark.asyncio
    async def test_sign_download_url_for_bucket(self) -> None:
        mock_api = MagicMock()
        mock_api.endpoint = "https://huggingface.co"
        backend = self._make_backend(mock_api)
        url = await backend.sign_download_url(
            "buckets/my-org/my-bucket/data/file.parquet"
        )
        assert url == (
            "https://huggingface.co/buckets/my-org/my-bucket/resolve/"
            "data/file.parquet"
        )

    @pytest.mark.asyncio
    async def test_read_range_zero_length(self) -> None:
        mock_api = MagicMock()
        mock_api.endpoint = "https://huggingface.co"
        backend = self._make_backend(mock_api)

        with patch("huggingface_hub.HfFileSystem") as mock_fs_cls:
            data = await backend.read_range(
                "datasets/scikit-learn/Fish/Fish.csv", offset=0, length=0
            )

        mock_fs_cls.assert_not_called()
        assert data == b""

    @pytest.mark.asyncio
    async def test_read_range(self) -> None:
        mock_api = MagicMock()
        mock_api.token = None
        mock_api.endpoint = "https://huggingface.co"
        backend = self._make_backend(mock_api)

        mock_file = MagicMock()
        mock_file.__enter__.return_value = mock_file
        mock_file.__exit__.return_value = False
        mock_file.read.return_value = b"partial-data"
        mock_fs = MagicMock()
        mock_fs.open.return_value = mock_file

        with patch(
            "huggingface_hub.HfFileSystem", return_value=mock_fs
        ) as mock_fs_cls:
            data = await backend.read_range(
                "datasets/scikit-learn/Fish/Fish.csv", offset=10, length=5
            )

        mock_fs_cls.assert_called_once_with(
            token=None, endpoint="https://huggingface.co"
        )
        # block_size=0 is required to get an exact read: fsspec's default
        # block cache always over-fetches past the requested range (see
        # `fsspec.caching.ReadAheadCache._fetch`).
        mock_fs.open.assert_called_once_with(
            "datasets/scikit-learn/Fish/Fish.csv", "rb", block_size=0
        )
        mock_file.seek.assert_called_once_with(10)
        mock_file.read.assert_called_once_with(5)
        assert data == b"partial-data"

    @pytest.mark.asyncio
    async def test_read_range_from_start(self) -> None:
        # offset=0 should not seek at all (avoids an unnecessary reconnect
        # in the underlying streaming file).
        mock_api = MagicMock()
        mock_api.token = None
        mock_api.endpoint = "https://huggingface.co"
        backend = self._make_backend(mock_api)

        mock_file = MagicMock()
        mock_file.__enter__.return_value = mock_file
        mock_file.__exit__.return_value = False
        mock_file.read.return_value = b"first-bytes"
        mock_fs = MagicMock()
        mock_fs.open.return_value = mock_file

        with patch("huggingface_hub.HfFileSystem", return_value=mock_fs):
            data = await backend.read_range(
                "datasets/scikit-learn/Fish/Fish.csv", length=1_000_000
            )

        mock_file.seek.assert_not_called()
        mock_file.read.assert_called_once_with(1_000_000)
        assert data == b"first-bytes"

    def test_list_entries_pagination(self) -> None:
        mock_api = MagicMock()
        mock_api.whoami.side_effect = Exception("no auth")
        mock_api.list_datasets.return_value = []
        mock_api.list_models.return_value = []
        mock_api.list_spaces.return_value = []
        mock_api.list_buckets.return_value = []

        entries = [
            StorageEntry(
                path=f"repo-{i}",
                kind="directory",
                size=0,
                last_modified=None,
            )
            for i in range(5)
        ]
        backend = self._make_backend(mock_api)
        with patch.object(
            backend,
            "_list_storage_entries",
            return_value=entries,
        ):
            result = backend.list_entries(prefix="", limit=2)
        assert result == snapshot(
            StorageListResult(
                entries=entries[:2],
                next_page_token="2",
            )
        )