Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stream the recipe batch merge instead of loading a dataframe (#16262)
Daniel Hanac6cf4b9e596
diff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.pyindex 37400865e91..c3f28b17057 100644--- a/studio/backend/core/data_recipe/jobs/worker.py+++ b/studio/backend/core/data_recipe/jobs/worker.py@@ -12,6 +12,8 @@ import shutilimport timeimport tracebackimport unicodedata+from collections import deque+from concurrent.futures import ThreadPoolExecutorfrom pathlib import Pathfrom typing import Any@@ -311,15 +313,17 @@ def _merge_batches_to_single_parquet(base_dataset_path: Path) -> None:except ImportError:return- dataframe = read_parquet_dataset(parquet_dir)# Written beside the shards first: a failed write must not take the generated batches with it.staged_file = base_dataset_path / "batch_00000.parquet.merging"try:- write_parquet_with_unified_types(- dataframe,- staged_file,- lambda frame: frame.to_parquet(staged_file, index = False),- )+ if not _stream_merge_batches(parquet_dir, parquet_files, staged_file):+ dataframe = read_parquet_dataset(parquet_dir)+ write_parquet_with_unified_types(+ dataframe,+ staged_file,+ lambda frame: frame.to_parquet(staged_file, index = False),+ )+ del dataframeexcept BaseException:staged_file.unlink(missing_ok = True)raise@@ -333,6 +337,102 @@ def _merge_batches_to_single_parquet(base_dataset_path: Path) -> None:)+# Peak memory ~ this buffer + shards in flight, not the whole dataset.+_MERGE_BUFFER_BYTES = 128 * 1024 * 1024+# write_table's default row group cap, as the dataframe path cut them.+_MERGE_ROW_GROUP_ROWS = 1024 * 1024+_MERGE_READ_AHEAD = 4+++def _stream_merge_batches(parquet_dir: Path, parquet_files: list[Path], staged_file: Path) -> bool:+ """Write the file the dataframe merge would, shard by shard; False (nothing written) when it could differ."""+ import pandas+ import pyarrow+ import pyarrow.parquet as pyarrow_parquet++ # The folder read skips "."/"_" names and reads everything else, subfolders included; the glob does not.+ if any(path.name.startswith((".", "_")) for path in parquet_files):+ return False+ listed = {path.name for path in parquet_files}+ for entry in parquet_dir.iterdir():+ if entry.name not in listed and not entry.name.startswith((".", "_")):+ return False++ try:+ schemas = [pyarrow_parquet.read_schema(path) for path in parquet_files]+ except Exception:+ # Let the folder read raise its own error naming the file.+ return False+ source_schema = schemas[0]+ for schema in schemas[1:]:+ if schema.names != source_schema.names or schema.types != source_schema.types:+ return False++ # Output schema + pandas metadata: the dataframe path run on zero rows.+ probe = pyarrow.BufferOutputStream()+ pyarrow_parquet.write_table(source_schema.empty_table(), probe)+ frame = pandas.read_parquet(pyarrow.BufferReader(probe.getvalue()), dtype_backend = "pyarrow")+ probe = pyarrow.BufferOutputStream()+ frame.to_parquet(probe, index = False)+ target_schema = pyarrow_parquet.read_schema(pyarrow.BufferReader(probe.getvalue()))+ del frame, probe+ if target_schema.names != source_schema.names or target_schema.types != source_schema.types:+ return False++ pending: list[Any] = []+ pending_rows = pending_bytes = 0+ writing = None+ # Write in the background while the next row group is read.+ with (+ pyarrow_parquet.ParquetWriter(staged_file, target_schema) as writer,+ ThreadPoolExecutor(max_workers = 1) as write_pool,+ ):++ def flush(final: bool) -> None:+ nonlocal pending, pending_rows, pending_bytes, writing+ if not pending:+ return+ table = pyarrow.concat_tables(pending)+ full = table.num_rows - table.num_rows % _MERGE_ROW_GROUP_ROWS+ if final or pending_bytes >= _MERGE_BUFFER_BYTES:+ full = table.num_rows+ if full:+ if writing is not None:+ writing.result()+ writing = write_pool.submit(+ writer.write_table, table.slice(0, full), row_group_size = _MERGE_ROW_GROUP_ROWS+ )+ rest = table.slice(full)+ pending = [rest] if rest.num_rows else []+ pending_rows, pending_bytes = rest.num_rows, rest.nbytes if rest.num_rows else 0++ use_threads = len(parquet_files) < _MERGE_READ_AHEAD++ def read(path: Path) -> Any:+ with pyarrow_parquet.ParquetFile(path) as shard:+ return shard.read(use_threads = use_threads)++ with ThreadPoolExecutor(max_workers = _MERGE_READ_AHEAD) as reader:+ upcoming = deque(+ reader.submit(read, path) for path in parquet_files[:_MERGE_READ_AHEAD]+ )+ for path in parquet_files[_MERGE_READ_AHEAD:] + [None] * len(upcoming):+ table = upcoming.popleft().result()+ if path is not None:+ upcoming.append(reader.submit(read, path))+ table = pyarrow.Table.from_arrays(table.columns, schema = target_schema)+ pending.append(table)+ pending_rows += table.num_rows+ pending_bytes += table.nbytes+ del table+ if pending_rows >= _MERGE_ROW_GROUP_ROWS or pending_bytes >= _MERGE_BUFFER_BYTES:+ flush(final = False)+ flush(final = True)+ if writing is not None:+ writing.result()+ return True++def _rewrite_merged_metadata(*, base_dataset_path: Path, parquet_file: Path) -> None:metadata_path = base_dataset_path / "metadata.json"if not metadata_path.exists():diff --git a/studio/backend/tests/test_data_recipe_streamed_merge.py b/studio/backend/tests/test_data_recipe_streamed_merge.pynew file mode 100644index 00000000000..d37f7d8821b--- /dev/null+++ b/studio/backend/tests/test_data_recipe_streamed_merge.py@@ -0,0 +1,253 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The batch merge streams shards into the merged file instead of loading the dataset as one dataframe.+Its output must stay the file the dataframe path wrote: same rows, order, Arrow schema and pandas metadata."""++import datetime+import shutil+import sys+import types+from pathlib import Path++import pytest++pytest.importorskip("pyarrow")++import pandas as pd+import pyarrow+import pyarrow.parquet as pq+++def _dataframe_read(path):+ return pd.read_parquet(path, dtype_backend = "pyarrow")++[email protected](name = "worker")+def fixture_worker(monkeypatch):+ try:+ import data_designer.config.utils.io_helpers # noqa: F401+ except ImportError:+ # Same read as data-designer's read_parquet_dataset for a folder whose shards agree.+ module = types.ModuleType("data_designer.config.utils.io_helpers")+ module.read_parquet_dataset = _dataframe_read+ for name in ("data_designer", "data_designer.config", "data_designer.config.utils"):+ monkeypatch.setitem(sys.modules, name, types.ModuleType(name))+ monkeypatch.setitem(sys.modules, module.__name__, module)+ from core.data_recipe.jobs import worker+ return worker+++def _row(n: int, shard: int, drift: bool) -> dict:+ row = {+ "id": n,+ "text": f"row {n} " * (1 + n % 4),+ "score": n / 3,+ "flag": n % 2 == 0,+ "when": datetime.datetime(2026, 1, 1) + datetime.timedelta(minutes = n),+ "tags": [f"t{n % 3}", f"u{n % 5}"][: n % 3],+ "obj": {"a": f"a{n}", "b": n % 7},+ "maybe": None if n % 3 == 0 else f"m{n}",+ }+ if drift:+ row["mixed"] = {"k": n} if shard % 2 == 0 else f'{{"k": {n}}}'+ row["count"] = n if shard % 3 else float(n)+ row["all_null"] = None if shard % 2 == 0 else f"v{n}"+ if shard % 4 == 1:+ del row["maybe"]+ return row+++def _write_dataset(base: Path, shards: int, rows: int, drift: bool) -> Path:+ parquet_dir = base / "parquet-files"+ parquet_dir.mkdir(parents = True)+ per = rows // shards+ for shard in range(shards):+ frame = pd.DataFrame([_row(shard * per + i, shard, drift) for i in range(per)])+ frame.to_parquet(parquet_dir / f"batch_{shard:05d}.parquet", index = False)+ return parquet_dir+++def _reference(parquet_dir: Path, scratch: Path) -> Path:+ """What the merge wrote before streaming: the whole folder as a pyarrow-backed dataframe, written back."""+ copy = scratch / "reference-input"+ shutil.copytree(parquet_dir, copy)+ out = scratch / "reference.parquet"+ _dataframe_read(copy).to_parquet(out, index = False)+ return out+++def _assert_same_file(actual: Path, expected: Path) -> None:+ actual_schema, expected_schema = pq.read_schema(actual), pq.read_schema(expected)+ assert actual_schema.equals(expected_schema, check_metadata = True)+ assert str(actual_schema) == str(expected_schema)+ actual_table, expected_table = pq.read_table(actual), pq.read_table(expected)+ assert actual_table.equals(expected_table)+ assert actual_table.to_pylist() == expected_table.to_pylist()+ assert pq.ParquetFile(actual).metadata.num_row_groups == (+ pq.ParquetFile(expected).metadata.num_row_groups+ )+ assert _dataframe_read(actual).equals(_dataframe_read(expected))++[email protected]("shards, rows", [(2, 10), (10, 200), (100, 1000)])[email protected]("drift", [False, True], ids = ["uniform", "drifting"])+def test_streamed_merge_writes_the_dataframe_merge_file(+ tmp_path: Path, worker, monkeypatch, shards, rows, drift+) -> None:+ base = tmp_path / "dataset"+ parquet_dir = _write_dataset(base, shards, rows, drift)+ worker._unify_batch_schemas(base)+ expected = _reference(parquet_dir, tmp_path)++ def no_dataframe(_path):+ raise AssertionError("the merge loaded the dataset as a dataframe")++ monkeypatch.setattr(+ sys.modules["data_designer.config.utils.io_helpers"], "read_parquet_dataset", no_dataframe+ )+ worker._merge_batches_to_single_parquet(base)++ assert sorted(p.name for p in parquet_dir.iterdir()) == ["batch_00000.parquet"]+ assert not list(base.glob("*.merging"))+ _assert_same_file(parquet_dir / "batch_00000.parquet", expected)+++def test_streamed_merge_buffers_a_bounded_amount(tmp_path: Path, worker, monkeypatch) -> None:+ base = tmp_path / "dataset"+ parquet_dir = _write_dataset(base, 10, 100, drift = False)+ expected = pq.read_table(_reference(parquet_dir, tmp_path))++ written: list[int] = []+ original = pq.ParquetWriter.write_table++ def counting(+ self,+ table,+ row_group_size = None,+ ):+ written.append(table.num_rows)+ return original(self, table, row_group_size = row_group_size)++ monkeypatch.setattr(pq.ParquetWriter, "write_table", counting)+ monkeypatch.setattr(worker, "_MERGE_BUFFER_BYTES", 1)+ worker._merge_batches_to_single_parquet(base)++ # The zero-row writes are the schema probe.+ assert [rows for rows in written if rows] == [10] * 10+ merged = pq.read_table(parquet_dir / "batch_00000.parquet")+ assert merged.equals(expected)+++def test_streamed_merge_cuts_row_groups_like_write_table(+ tmp_path: Path, worker, monkeypatch+) -> None:+ base = tmp_path / "dataset"+ parquet_dir = _write_dataset(base, 10, 50, drift = False)+ expected = pq.read_table(_reference(parquet_dir, tmp_path))+ monkeypatch.setattr(worker, "_MERGE_ROW_GROUP_ROWS", 7)+ worker._merge_batches_to_single_parquet(base)++ metadata = pq.ParquetFile(parquet_dir / "batch_00000.parquet").metadata+ assert [metadata.row_group(i).num_rows for i in range(metadata.num_row_groups)] == [7] * 7 + [1]+ assert pq.read_table(parquet_dir / "batch_00000.parquet").equals(expected)+++def test_shards_whose_types_differ_take_the_dataframe_merge(tmp_path: Path, worker) -> None:+ # Unify could not settle the types (it logs and keeps them): the dataframe path still decides the merge.+ base = tmp_path / "dataset"+ parquet_dir = base / "parquet-files"+ parquet_dir.mkdir(parents = True)+ pd.DataFrame({"n": [1, 2]}).to_parquet(parquet_dir / "batch_00000.parquet", index = False)+ pd.DataFrame({"n": [1.5]}).to_parquet(parquet_dir / "batch_00001.parquet", index = False)+ calls = []+ io_helpers = sys.modules["data_designer.config.utils.io_helpers"]+ real = io_helpers.read_parquet_dataset++ def recording(path):+ calls.append(path)+ return pd.concat(+ [_dataframe_read(p) for p in sorted(Path(path).glob("*.parquet"))], ignore_index = True+ )++ io_helpers.read_parquet_dataset = recording+ try:+ worker._merge_batches_to_single_parquet(base)+ finally:+ io_helpers.read_parquet_dataset = real++ assert calls == [parquet_dir]+ assert pq.read_table(parquet_dir / "batch_00000.parquet").column("n").to_pylist() == [1, 2, 1.5]+++def test_a_stray_file_the_folder_read_would_pick_up_takes_the_dataframe_merge(+ tmp_path: Path, worker+) -> None:+ base = tmp_path / "dataset"+ parquet_dir = _write_dataset(base, 2, 4, drift = False)+ stray = parquet_dir / "batch_00002.parquet.unifying"+ pq.write_table(pq.read_table(parquet_dir / "batch_00001.parquet"), stray)+ expected = pq.read_table(_reference(parquet_dir, tmp_path))+ worker._merge_batches_to_single_parquet(base)++ assert pq.read_table(parquet_dir / "batch_00000.parquet").equals(expected)+ assert expected.num_rows == 6++[email protected]("name", ["_partial.parquet", ".partial.parquet"])+def test_a_shard_name_the_folder_read_skips_stays_out_of_the_merge(+ tmp_path: Path, worker, name+) -> None:+ # The glob lists "_" and "." names, the folder read skips them: the merge must not take their rows.+ base = tmp_path / "dataset"+ parquet_dir = _write_dataset(base, 2, 4, drift = False)+ pq.write_table(pq.read_table(parquet_dir / "batch_00001.parquet"), parquet_dir / name)+ expected = pq.read_table(_reference(parquet_dir, tmp_path))+ worker._merge_batches_to_single_parquet(base)++ assert expected.num_rows == 4+ assert pq.read_table(parquet_dir / "batch_00000.parquet").equals(expected)+++def test_an_unreadable_shard_fails_the_merge_as_the_folder_read_does(+ tmp_path: Path, worker+) -> None:+ base = tmp_path / "dataset"+ parquet_dir = _write_dataset(base, 2, 4, drift = False)+ (parquet_dir / "batch_00001.parquet").write_bytes(b"PAR1 not a parquet file")+ with pytest.raises(pyarrow.ArrowInvalid, match = "batch_00001.parquet"):+ worker._merge_batches_to_single_parquet(base)+ assert sorted(p.name for p in parquet_dir.iterdir()) == [+ "batch_00000.parquet",+ "batch_00001.parquet",+ ]+ assert not list(base.glob("*.merging"))+++def test_a_failed_background_write_fails_the_merge_and_keeps_the_shards(+ tmp_path: Path, worker, monkeypatch+) -> None:+ base = tmp_path / "dataset"+ parquet_dir = _write_dataset(base, 10, 100, drift = False)+ before = {p.name: p.read_bytes() for p in parquet_dir.iterdir()}+ original = pq.ParquetWriter.write_table+ calls: list[int] = []++ def failing(+ self,+ table,+ row_group_size = None,+ ):+ if table.num_rows:+ calls.append(table.num_rows)+ if len(calls) == 3:+ raise OSError("disk full")+ return original(self, table, row_group_size = row_group_size)++ monkeypatch.setattr(pq.ParquetWriter, "write_table", failing)+ monkeypatch.setattr(worker, "_MERGE_BUFFER_BYTES", 1)+ with pytest.raises(OSError, match = "disk full"):+ worker._merge_batches_to_single_parquet(base)++ assert {p.name: p.read_bytes() for p in parquet_dir.iterdir()} == before+ assert not list(base.glob("*.merging"))