Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: preview local Parquet from its leading rows instead of loading it whole (#15868)

Daniel Han8678bf729b36
diff --git a/studio/backend/hub/services/datasets/local.py b/studio/backend/hub/services/datasets/local.py
index db52f97b069..debc882c73f 100644
--- a/studio/backend/hub/services/datasets/local.py
+++ b/studio/backend/hub/services/datasets/local.py
@@ -289,6 +289,58 @@ def _stream_file_preview_slice(path: Path, preview_size: int, **load_kwargs):
return Dataset.from_list(rows, features = load_kwargs.get("features")), None
+def _parquet_preview_slice(files: list[Path], preview_size: int):
+ """Leading ``preview_size`` rows + footer row count, typed as the datasets Parquet builder types them,
+ without the full load's cache write. ``None`` (full load decides) on differing schemas, no rows, or any error."""
+ try:
+ import pyarrow as pa
+ import pyarrow.parquet as pq
+ from datasets import Dataset, DatasetInfo, Features
+ from datasets.table import InMemoryTable, table_cast
+
+ schema, total_rows = None, 0
+ for path in files:
+ metadata = pq.read_metadata(path)
+ file_schema = metadata.schema.to_arrow_schema()
+ if schema is None:
+ schema = file_schema
+ elif not file_schema.equals(schema, check_metadata = True):
+ # The full load casts later files to the first one's features or raises; leave both to it.
+ return None
+ total_rows += metadata.num_rows
+ if total_rows <= 0:
+ return None
+
+ features = Features.from_arrow_schema(schema)
+ arrow_schema = features.arrow_schema
+ wanted = min(preview_size, total_rows)
+ tables, decoded_rows = [], 0
+ # Decode every page (damage past the preview rows must still fail), one row group per batch at most.
+ for path in files:
+ # A file object, so the handle is closed here (Windows would keep the file locked otherwise).
+ with open(path, "rb") as handle:
+ parquet_file = pq.ParquetFile(handle, pre_buffer = False, buffer_size = 1 << 20)
+ metadata = parquet_file.metadata
+ for index in range(metadata.num_row_groups):
+ batch_size = max(1, min(8192, metadata.row_group(index).num_rows))
+ for batch in parquet_file.iter_batches(
+ batch_size = batch_size, row_groups = [index], use_threads = False
+ ):
+ decoded_rows += batch.num_rows
+ if wanted > 0:
+ table = pa.Table.from_batches([batch.slice(0, wanted)])
+ tables.append(table_cast(table, arrow_schema))
+ wanted -= tables[-1].num_rows
+ if decoded_rows != total_rows or wanted > 0:
+ return None
+ table = pa.concat_tables(tables).combine_chunks()
+ # The full load writes its rows under the features' own Arrow schema (list child names included).
+ table = table.cast(arrow_schema)
+ return Dataset(InMemoryTable(table), info = DatasetInfo(features = features)), total_rows
+ except Exception:
+ return None
+
+
def _load_local_preview_slice(*, dataset_path: Path, train_split: str, preview_size: int):
# Non-streaming loads take the cached builder lock; use the EACCES-safe wrapper.
from utils.datasets.cache_safe import load_dataset_cache_safe as load_dataset
@@ -302,6 +354,10 @@ def _load_local_preview_slice(*, dataset_path: Path, train_split: str, preview_s
)
parquet_files = drop_appledouble_metadata(sorted(parquet_dir.glob("*.parquet")))
if parquet_files:
+ if train_split == "train":
+ preview = _parquet_preview_slice(parquet_files, preview_size)
+ if preview is not None:
+ return preview
dataset = load_dataset(
"parquet",
data_files = [str(path) for path in parquet_files],
@@ -324,6 +380,10 @@ def _load_local_preview_slice(*, dataset_path: Path, train_split: str, preview_s
suffix = dataset_path.suffix.lower()
# Parquet/Arrow give a cheap exact total_rows; JSON/CSV carry none, so stream and report None.
if suffix == ".parquet":
+ if train_split == "train":
+ preview = _parquet_preview_slice([dataset_path], preview_size)
+ if preview is not None:
+ return preview
dataset = load_dataset("parquet", data_files = str(dataset_path), split = train_split)
total_rows = len(dataset)
preview_slice = dataset.select(range(min(preview_size, total_rows)))
diff --git a/studio/backend/tests/test_parquet_preview_leading_rows.py b/studio/backend/tests/test_parquet_preview_leading_rows.py
new file mode 100644
index 00000000000..8de9238d877
--- /dev/null
+++ b/studio/backend/tests/test_parquet_preview_leading_rows.py
@@ -0,0 +1,194 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A local Parquet preview reads only its leading rows, and shows what loading the files whole shows."""
+
+from __future__ import annotations
+
+import datetime as dt
+import decimal
+import io
+import sys
+from pathlib import Path
+
+_BACKEND = Path(__file__).resolve().parent.parent
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+import pyarrow as pa # noqa: E402
+import pyarrow.parquet as pq # noqa: E402
+import pytest # noqa: E402
+
+import hub.services.datasets.local as local # noqa: E402
+import utils.datasets.cache_safe as cache_safe # noqa: E402
+
+
[email protected](autouse = True)
+def _own_datasets_cache(tmp_path, monkeypatch):
+ monkeypatch.setenv("HF_DATASETS_CACHE", str(tmp_path / "hf_datasets_cache"))
+ import datasets.config
+ monkeypatch.setattr(datasets.config, "HF_DATASETS_CACHE", tmp_path / "hf_datasets_cache")
+
+
+def _jpeg(shade: int) -> bytes:
+ from PIL import Image
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (8, 8), (shade, 0, 255 - shade)).save(buffer, format = "JPEG")
+ return buffer.getvalue()
+
+
+def _typed_table(rows: int) -> pa.Table:
+ return pa.table(
+ {
+ "instruction": [f"q{i}" if i % 4 else None for i in range(rows)],
+ "messages": [
+ [{"role": "user", "content": f"u{i}"}, {"role": "assistant", "content": None}]
+ for i in range(rows)
+ ],
+ "ts": pa.array(
+ [dt.datetime(2024, 1, 1, 0, 0, i % 60) for i in range(rows)],
+ pa.timestamp("us", tz = "UTC"),
+ ),
+ "price": pa.array(
+ [decimal.Decimal(f"{i}.25") for i in range(rows)], pa.decimal128(9, 2)
+ ),
+ "tag": pa.array([f"t{i % 3}" for i in range(rows)]).dictionary_encode(),
+ "long": pa.array([f"L{i}" for i in range(rows)], pa.large_string()),
+ "nested": [{"a": i, "b": [f"x{i}"]} for i in range(rows)],
+ }
+ )
+
+
+def _image_file(path: Path, rows: int) -> None:
+ from datasets import Features, Image, Value
+
+ features = Features({"image": Image(), "text": Value("string")})
+ table = pa.table(
+ {
+ "image": [{"bytes": _jpeg(i * 9 % 256), "path": f"{i}.jpg"} for i in range(rows)],
+ "text": [f"caption {i}" for i in range(rows)],
+ },
+ schema = features.arrow_schema,
+ )
+ pq.write_table(table, path, row_group_size = 4)
+
+
+_LEADING_ROWS = getattr(local, "_parquet_preview_slice", None)
+
+
+def _preview(
+ path: Path,
+ monkeypatch,
+ *,
+ whole: bool,
+ split: str = "train",
+):
+ leading_rows = (lambda files, size: None) if whole else _LEADING_ROWS
+ monkeypatch.setattr(local, "_parquet_preview_slice", leading_rows, raising = False)
+ try:
+ dataset, total = local._load_local_preview_slice(
+ dataset_path = path, train_split = split, preview_size = 10
+ )
+ except Exception as error: # noqa: BLE001 - the error itself is compared
+ return ("error", type(error).__name__, str(error))
+ table = dataset.with_format("arrow")[:]
+ return total, repr(dataset.features), table.schema, table.to_pylist()
+
+
+def _fixtures(root: Path) -> list[Path]:
+ pq.write_table(_typed_table(37), root / "typed.parquet", row_group_size = 6)
+ _image_file(root / "images.parquet", 13)
+ pq.write_table(pa.table({"text": ["a", "b", "c"]}), root / "short.parquet")
+ shards = root / "shards"
+ shards.mkdir()
+ for index in range(5):
+ pq.write_table(_typed_table(index % 3 + 5), shards / f"part-{index:05d}.parquet")
+ recipe = root / "recipe" / "parquet-files"
+ recipe.mkdir(parents = True)
+ for index in range(3):
+ pq.write_table(
+ pa.table({"prompt": [f"p{index}{i}" for i in range(4)]}),
+ recipe / f"batch_{index}.parquet",
+ )
+ return [
+ root / "typed.parquet",
+ root / "images.parquet",
+ root / "short.parquet",
+ shards,
+ root / "recipe",
+ ]
+
+
+def test_parquet_preview_matches_the_whole_load(tmp_path, monkeypatch):
+ for path in _fixtures(tmp_path):
+ whole = _preview(path, monkeypatch, whole = True)
+ leading = _preview(path, monkeypatch, whole = False)
+ assert whole[0] != "error", whole
+ assert leading == whole, path.name
+
+
+def test_parquet_preview_does_not_load_the_files_whole(tmp_path, monkeypatch):
+ calls = []
+ real = cache_safe.load_dataset_cache_safe
+
+ def counting(*args, **kwargs):
+ calls.append(kwargs.get("data_files"))
+ return real(*args, **kwargs)
+
+ monkeypatch.setattr(cache_safe, "load_dataset_cache_safe", counting)
+ for path in _fixtures(tmp_path):
+ dataset, total = local._load_local_preview_slice(
+ dataset_path = path, train_split = "train", preview_size = 10
+ )
+ assert len(dataset) == min(10, total)
+ assert calls == []
+ assert not (tmp_path / "hf_datasets_cache").exists() or not any(
+ (tmp_path / "hf_datasets_cache").rglob("*.arrow")
+ )
+
+
+def test_parquet_preview_still_fails_on_data_damaged_past_the_preview_rows(tmp_path, monkeypatch):
+ damaged = tmp_path / "damaged.parquet"
+ pq.write_table(_typed_table(60), damaged, row_group_size = 10)
+ column = pq.read_metadata(damaged).row_group(5).column(0)
+ with open(damaged, "r+b") as handle:
+ handle.seek(column.data_page_offset)
+ handle.write(b"\0" * 64)
+ assert _LEADING_ROWS([damaged], 10) is None
+ whole = _preview(damaged, monkeypatch, whole = True)
+ leading = _preview(damaged, monkeypatch, whole = False)
+ assert whole[0] == "error", whole
+ assert leading == whole
+
+
+def test_parquet_preview_leaves_uncertain_inputs_to_the_whole_load(tmp_path, monkeypatch):
+ mixed = tmp_path / "mixed"
+ mixed.mkdir()
+ pq.write_table(pa.table({"text": ["x", "y"]}), mixed / "a.parquet")
+ pq.write_table(pa.table({"text": [1, 2]}), mixed / "b.parquet")
+ renamed = tmp_path / "renamed"
+ renamed.mkdir()
+ pq.write_table(pa.table({"text": ["x"]}), renamed / "a.parquet")
+ pq.write_table(pa.table({"other": ["y"]}), renamed / "b.parquet")
+ empty = tmp_path / "empty.parquet"
+ pq.write_table(pa.table({"text": pa.array([], pa.string())}), empty)
+ broken = tmp_path / "broken.parquet"
+ broken.write_bytes(b"not parquet" * 8)
+ typed = tmp_path / "typed.parquet"
+ pq.write_table(_typed_table(12), typed)
+
+ cases = [
+ (mixed, "train"),
+ (renamed, "train"),
+ (empty, "train"),
+ (broken, "train"),
+ (typed, "test"),
+ ]
+ for path, split in cases:
+ if split == "train":
+ files = sorted(path.glob("*.parquet")) if path.is_dir() else [path]
+ assert _LEADING_ROWS(files, 10) is None, path.name
+ whole = _preview(path, monkeypatch, whole = True, split = split)
+ leading = _preview(path, monkeypatch, whole = False, split = split)
+ assert leading == whole, (path.name, split)