Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: preview local Parquet from its leading rows instead of loading it whole (#15868)
Daniel Han8678bf729b36
diff --git a/studio/backend/hub/services/datasets/local.py b/studio/backend/hub/services/datasets/local.pyindex db52f97b069..debc882c73f 100644--- a/studio/backend/hub/services/datasets/local.py+++ b/studio/backend/hub/services/datasets/local.py@@ -289,6 +289,58 @@ def _stream_file_preview_slice(path: Path, preview_size: int, **load_kwargs):return Dataset.from_list(rows, features = load_kwargs.get("features")), None+def _parquet_preview_slice(files: list[Path], preview_size: int):+ """Leading ``preview_size`` rows + footer row count, typed as the datasets Parquet builder types them,+ without the full load's cache write. ``None`` (full load decides) on differing schemas, no rows, or any error."""+ try:+ import pyarrow as pa+ import pyarrow.parquet as pq+ from datasets import Dataset, DatasetInfo, Features+ from datasets.table import InMemoryTable, table_cast++ schema, total_rows = None, 0+ for path in files:+ metadata = pq.read_metadata(path)+ file_schema = metadata.schema.to_arrow_schema()+ if schema is None:+ schema = file_schema+ elif not file_schema.equals(schema, check_metadata = True):+ # The full load casts later files to the first one's features or raises; leave both to it.+ return None+ total_rows += metadata.num_rows+ if total_rows <= 0:+ return None++ features = Features.from_arrow_schema(schema)+ arrow_schema = features.arrow_schema+ wanted = min(preview_size, total_rows)+ tables, decoded_rows = [], 0+ # Decode every page (damage past the preview rows must still fail), one row group per batch at most.+ for path in files:+ # A file object, so the handle is closed here (Windows would keep the file locked otherwise).+ with open(path, "rb") as handle:+ parquet_file = pq.ParquetFile(handle, pre_buffer = False, buffer_size = 1 << 20)+ metadata = parquet_file.metadata+ for index in range(metadata.num_row_groups):+ batch_size = max(1, min(8192, metadata.row_group(index).num_rows))+ for batch in parquet_file.iter_batches(+ batch_size = batch_size, row_groups = [index], use_threads = False+ ):+ decoded_rows += batch.num_rows+ if wanted > 0:+ table = pa.Table.from_batches([batch.slice(0, wanted)])+ tables.append(table_cast(table, arrow_schema))+ wanted -= tables[-1].num_rows+ if decoded_rows != total_rows or wanted > 0:+ return None+ table = pa.concat_tables(tables).combine_chunks()+ # The full load writes its rows under the features' own Arrow schema (list child names included).+ table = table.cast(arrow_schema)+ return Dataset(InMemoryTable(table), info = DatasetInfo(features = features)), total_rows+ except Exception:+ return None++def _load_local_preview_slice(*, dataset_path: Path, train_split: str, preview_size: int):# Non-streaming loads take the cached builder lock; use the EACCES-safe wrapper.from utils.datasets.cache_safe import load_dataset_cache_safe as load_dataset@@ -302,6 +354,10 @@ def _load_local_preview_slice(*, dataset_path: Path, train_split: str, preview_s)parquet_files = drop_appledouble_metadata(sorted(parquet_dir.glob("*.parquet")))if parquet_files:+ if train_split == "train":+ preview = _parquet_preview_slice(parquet_files, preview_size)+ if preview is not None:+ return previewdataset = load_dataset("parquet",data_files = [str(path) for path in parquet_files],@@ -324,6 +380,10 @@ def _load_local_preview_slice(*, dataset_path: Path, train_split: str, preview_ssuffix = dataset_path.suffix.lower()# Parquet/Arrow give a cheap exact total_rows; JSON/CSV carry none, so stream and report None.if suffix == ".parquet":+ if train_split == "train":+ preview = _parquet_preview_slice([dataset_path], preview_size)+ if preview is not None:+ return previewdataset = load_dataset("parquet", data_files = str(dataset_path), split = train_split)total_rows = len(dataset)preview_slice = dataset.select(range(min(preview_size, total_rows)))diff --git a/studio/backend/tests/test_parquet_preview_leading_rows.py b/studio/backend/tests/test_parquet_preview_leading_rows.pynew file mode 100644index 00000000000..8de9238d877--- /dev/null+++ b/studio/backend/tests/test_parquet_preview_leading_rows.py@@ -0,0 +1,194 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A local Parquet preview reads only its leading rows, and shows what loading the files whole shows."""++from __future__ import annotations++import datetime as dt+import decimal+import io+import sys+from pathlib import Path++_BACKEND = Path(__file__).resolve().parent.parent+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++import pyarrow as pa # noqa: E402+import pyarrow.parquet as pq # noqa: E402+import pytest # noqa: E402++import hub.services.datasets.local as local # noqa: E402+import utils.datasets.cache_safe as cache_safe # noqa: E402++[email protected](autouse = True)+def _own_datasets_cache(tmp_path, monkeypatch):+ monkeypatch.setenv("HF_DATASETS_CACHE", str(tmp_path / "hf_datasets_cache"))+ import datasets.config+ monkeypatch.setattr(datasets.config, "HF_DATASETS_CACHE", tmp_path / "hf_datasets_cache")+++def _jpeg(shade: int) -> bytes:+ from PIL import Image++ buffer = io.BytesIO()+ Image.new("RGB", (8, 8), (shade, 0, 255 - shade)).save(buffer, format = "JPEG")+ return buffer.getvalue()+++def _typed_table(rows: int) -> pa.Table:+ return pa.table(+ {+ "instruction": [f"q{i}" if i % 4 else None for i in range(rows)],+ "messages": [+ [{"role": "user", "content": f"u{i}"}, {"role": "assistant", "content": None}]+ for i in range(rows)+ ],+ "ts": pa.array(+ [dt.datetime(2024, 1, 1, 0, 0, i % 60) for i in range(rows)],+ pa.timestamp("us", tz = "UTC"),+ ),+ "price": pa.array(+ [decimal.Decimal(f"{i}.25") for i in range(rows)], pa.decimal128(9, 2)+ ),+ "tag": pa.array([f"t{i % 3}" for i in range(rows)]).dictionary_encode(),+ "long": pa.array([f"L{i}" for i in range(rows)], pa.large_string()),+ "nested": [{"a": i, "b": [f"x{i}"]} for i in range(rows)],+ }+ )+++def _image_file(path: Path, rows: int) -> None:+ from datasets import Features, Image, Value++ features = Features({"image": Image(), "text": Value("string")})+ table = pa.table(+ {+ "image": [{"bytes": _jpeg(i * 9 % 256), "path": f"{i}.jpg"} for i in range(rows)],+ "text": [f"caption {i}" for i in range(rows)],+ },+ schema = features.arrow_schema,+ )+ pq.write_table(table, path, row_group_size = 4)+++_LEADING_ROWS = getattr(local, "_parquet_preview_slice", None)+++def _preview(+ path: Path,+ monkeypatch,+ *,+ whole: bool,+ split: str = "train",+):+ leading_rows = (lambda files, size: None) if whole else _LEADING_ROWS+ monkeypatch.setattr(local, "_parquet_preview_slice", leading_rows, raising = False)+ try:+ dataset, total = local._load_local_preview_slice(+ dataset_path = path, train_split = split, preview_size = 10+ )+ except Exception as error: # noqa: BLE001 - the error itself is compared+ return ("error", type(error).__name__, str(error))+ table = dataset.with_format("arrow")[:]+ return total, repr(dataset.features), table.schema, table.to_pylist()+++def _fixtures(root: Path) -> list[Path]:+ pq.write_table(_typed_table(37), root / "typed.parquet", row_group_size = 6)+ _image_file(root / "images.parquet", 13)+ pq.write_table(pa.table({"text": ["a", "b", "c"]}), root / "short.parquet")+ shards = root / "shards"+ shards.mkdir()+ for index in range(5):+ pq.write_table(_typed_table(index % 3 + 5), shards / f"part-{index:05d}.parquet")+ recipe = root / "recipe" / "parquet-files"+ recipe.mkdir(parents = True)+ for index in range(3):+ pq.write_table(+ pa.table({"prompt": [f"p{index}{i}" for i in range(4)]}),+ recipe / f"batch_{index}.parquet",+ )+ return [+ root / "typed.parquet",+ root / "images.parquet",+ root / "short.parquet",+ shards,+ root / "recipe",+ ]+++def test_parquet_preview_matches_the_whole_load(tmp_path, monkeypatch):+ for path in _fixtures(tmp_path):+ whole = _preview(path, monkeypatch, whole = True)+ leading = _preview(path, monkeypatch, whole = False)+ assert whole[0] != "error", whole+ assert leading == whole, path.name+++def test_parquet_preview_does_not_load_the_files_whole(tmp_path, monkeypatch):+ calls = []+ real = cache_safe.load_dataset_cache_safe++ def counting(*args, **kwargs):+ calls.append(kwargs.get("data_files"))+ return real(*args, **kwargs)++ monkeypatch.setattr(cache_safe, "load_dataset_cache_safe", counting)+ for path in _fixtures(tmp_path):+ dataset, total = local._load_local_preview_slice(+ dataset_path = path, train_split = "train", preview_size = 10+ )+ assert len(dataset) == min(10, total)+ assert calls == []+ assert not (tmp_path / "hf_datasets_cache").exists() or not any(+ (tmp_path / "hf_datasets_cache").rglob("*.arrow")+ )+++def test_parquet_preview_still_fails_on_data_damaged_past_the_preview_rows(tmp_path, monkeypatch):+ damaged = tmp_path / "damaged.parquet"+ pq.write_table(_typed_table(60), damaged, row_group_size = 10)+ column = pq.read_metadata(damaged).row_group(5).column(0)+ with open(damaged, "r+b") as handle:+ handle.seek(column.data_page_offset)+ handle.write(b"\0" * 64)+ assert _LEADING_ROWS([damaged], 10) is None+ whole = _preview(damaged, monkeypatch, whole = True)+ leading = _preview(damaged, monkeypatch, whole = False)+ assert whole[0] == "error", whole+ assert leading == whole+++def test_parquet_preview_leaves_uncertain_inputs_to_the_whole_load(tmp_path, monkeypatch):+ mixed = tmp_path / "mixed"+ mixed.mkdir()+ pq.write_table(pa.table({"text": ["x", "y"]}), mixed / "a.parquet")+ pq.write_table(pa.table({"text": [1, 2]}), mixed / "b.parquet")+ renamed = tmp_path / "renamed"+ renamed.mkdir()+ pq.write_table(pa.table({"text": ["x"]}), renamed / "a.parquet")+ pq.write_table(pa.table({"other": ["y"]}), renamed / "b.parquet")+ empty = tmp_path / "empty.parquet"+ pq.write_table(pa.table({"text": pa.array([], pa.string())}), empty)+ broken = tmp_path / "broken.parquet"+ broken.write_bytes(b"not parquet" * 8)+ typed = tmp_path / "typed.parquet"+ pq.write_table(_typed_table(12), typed)++ cases = [+ (mixed, "train"),+ (renamed, "train"),+ (empty, "train"),+ (broken, "train"),+ (typed, "test"),+ ]+ for path, split in cases:+ if split == "train":+ files = sorted(path.glob("*.parquet")) if path.is_dir() else [path]+ assert _LEADING_ROWS(files, 10) is None, path.name+ whole = _preview(path, monkeypatch, whole = True, split = split)+ leading = _preview(path, monkeypatch, whole = False, split = split)+ assert leading == whole, (path.name, split)