Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stream the recipe batch merge instead of loading a dataframe (#16262)

Daniel Hanac6cf4b9e596
diff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.py
index 37400865e91..c3f28b17057 100644
--- a/studio/backend/core/data_recipe/jobs/worker.py
+++ b/studio/backend/core/data_recipe/jobs/worker.py
@@ -12,6 +12,8 @@ import shutil
import time
import traceback
import unicodedata
+from collections import deque
+from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any
@@ -311,15 +313,17 @@ def _merge_batches_to_single_parquet(base_dataset_path: Path) -> None:
except ImportError:
return
- dataframe = read_parquet_dataset(parquet_dir)
# Written beside the shards first: a failed write must not take the generated batches with it.
staged_file = base_dataset_path / "batch_00000.parquet.merging"
try:
- write_parquet_with_unified_types(
- dataframe,
- staged_file,
- lambda frame: frame.to_parquet(staged_file, index = False),
- )
+ if not _stream_merge_batches(parquet_dir, parquet_files, staged_file):
+ dataframe = read_parquet_dataset(parquet_dir)
+ write_parquet_with_unified_types(
+ dataframe,
+ staged_file,
+ lambda frame: frame.to_parquet(staged_file, index = False),
+ )
+ del dataframe
except BaseException:
staged_file.unlink(missing_ok = True)
raise
@@ -333,6 +337,102 @@ def _merge_batches_to_single_parquet(base_dataset_path: Path) -> None:
)
+# Peak memory ~ this buffer + shards in flight, not the whole dataset.
+_MERGE_BUFFER_BYTES = 128 * 1024 * 1024
+# write_table's default row group cap, as the dataframe path cut them.
+_MERGE_ROW_GROUP_ROWS = 1024 * 1024
+_MERGE_READ_AHEAD = 4
+
+
+def _stream_merge_batches(parquet_dir: Path, parquet_files: list[Path], staged_file: Path) -> bool:
+ """Write the file the dataframe merge would, shard by shard; False (nothing written) when it could differ."""
+ import pandas
+ import pyarrow
+ import pyarrow.parquet as pyarrow_parquet
+
+ # The folder read skips "."/"_" names and reads everything else, subfolders included; the glob does not.
+ if any(path.name.startswith((".", "_")) for path in parquet_files):
+ return False
+ listed = {path.name for path in parquet_files}
+ for entry in parquet_dir.iterdir():
+ if entry.name not in listed and not entry.name.startswith((".", "_")):
+ return False
+
+ try:
+ schemas = [pyarrow_parquet.read_schema(path) for path in parquet_files]
+ except Exception:
+ # Let the folder read raise its own error naming the file.
+ return False
+ source_schema = schemas[0]
+ for schema in schemas[1:]:
+ if schema.names != source_schema.names or schema.types != source_schema.types:
+ return False
+
+ # Output schema + pandas metadata: the dataframe path run on zero rows.
+ probe = pyarrow.BufferOutputStream()
+ pyarrow_parquet.write_table(source_schema.empty_table(), probe)
+ frame = pandas.read_parquet(pyarrow.BufferReader(probe.getvalue()), dtype_backend = "pyarrow")
+ probe = pyarrow.BufferOutputStream()
+ frame.to_parquet(probe, index = False)
+ target_schema = pyarrow_parquet.read_schema(pyarrow.BufferReader(probe.getvalue()))
+ del frame, probe
+ if target_schema.names != source_schema.names or target_schema.types != source_schema.types:
+ return False
+
+ pending: list[Any] = []
+ pending_rows = pending_bytes = 0
+ writing = None
+ # Write in the background while the next row group is read.
+ with (
+ pyarrow_parquet.ParquetWriter(staged_file, target_schema) as writer,
+ ThreadPoolExecutor(max_workers = 1) as write_pool,
+ ):
+
+ def flush(final: bool) -> None:
+ nonlocal pending, pending_rows, pending_bytes, writing
+ if not pending:
+ return
+ table = pyarrow.concat_tables(pending)
+ full = table.num_rows - table.num_rows % _MERGE_ROW_GROUP_ROWS
+ if final or pending_bytes >= _MERGE_BUFFER_BYTES:
+ full = table.num_rows
+ if full:
+ if writing is not None:
+ writing.result()
+ writing = write_pool.submit(
+ writer.write_table, table.slice(0, full), row_group_size = _MERGE_ROW_GROUP_ROWS
+ )
+ rest = table.slice(full)
+ pending = [rest] if rest.num_rows else []
+ pending_rows, pending_bytes = rest.num_rows, rest.nbytes if rest.num_rows else 0
+
+ use_threads = len(parquet_files) < _MERGE_READ_AHEAD
+
+ def read(path: Path) -> Any:
+ with pyarrow_parquet.ParquetFile(path) as shard:
+ return shard.read(use_threads = use_threads)
+
+ with ThreadPoolExecutor(max_workers = _MERGE_READ_AHEAD) as reader:
+ upcoming = deque(
+ reader.submit(read, path) for path in parquet_files[:_MERGE_READ_AHEAD]
+ )
+ for path in parquet_files[_MERGE_READ_AHEAD:] + [None] * len(upcoming):
+ table = upcoming.popleft().result()
+ if path is not None:
+ upcoming.append(reader.submit(read, path))
+ table = pyarrow.Table.from_arrays(table.columns, schema = target_schema)
+ pending.append(table)
+ pending_rows += table.num_rows
+ pending_bytes += table.nbytes
+ del table
+ if pending_rows >= _MERGE_ROW_GROUP_ROWS or pending_bytes >= _MERGE_BUFFER_BYTES:
+ flush(final = False)
+ flush(final = True)
+ if writing is not None:
+ writing.result()
+ return True
+
+
def _rewrite_merged_metadata(*, base_dataset_path: Path, parquet_file: Path) -> None:
metadata_path = base_dataset_path / "metadata.json"
if not metadata_path.exists():
diff --git a/studio/backend/tests/test_data_recipe_streamed_merge.py b/studio/backend/tests/test_data_recipe_streamed_merge.py
new file mode 100644
index 00000000000..d37f7d8821b
--- /dev/null
+++ b/studio/backend/tests/test_data_recipe_streamed_merge.py
@@ -0,0 +1,253 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The batch merge streams shards into the merged file instead of loading the dataset as one dataframe.
+Its output must stay the file the dataframe path wrote: same rows, order, Arrow schema and pandas metadata."""
+
+import datetime
+import shutil
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+pytest.importorskip("pyarrow")
+
+import pandas as pd
+import pyarrow
+import pyarrow.parquet as pq
+
+
+def _dataframe_read(path):
+ return pd.read_parquet(path, dtype_backend = "pyarrow")
+
+
[email protected](name = "worker")
+def fixture_worker(monkeypatch):
+ try:
+ import data_designer.config.utils.io_helpers # noqa: F401
+ except ImportError:
+ # Same read as data-designer's read_parquet_dataset for a folder whose shards agree.
+ module = types.ModuleType("data_designer.config.utils.io_helpers")
+ module.read_parquet_dataset = _dataframe_read
+ for name in ("data_designer", "data_designer.config", "data_designer.config.utils"):
+ monkeypatch.setitem(sys.modules, name, types.ModuleType(name))
+ monkeypatch.setitem(sys.modules, module.__name__, module)
+ from core.data_recipe.jobs import worker
+ return worker
+
+
+def _row(n: int, shard: int, drift: bool) -> dict:
+ row = {
+ "id": n,
+ "text": f"row {n} " * (1 + n % 4),
+ "score": n / 3,
+ "flag": n % 2 == 0,
+ "when": datetime.datetime(2026, 1, 1) + datetime.timedelta(minutes = n),
+ "tags": [f"t{n % 3}", f"u{n % 5}"][: n % 3],
+ "obj": {"a": f"a{n}", "b": n % 7},
+ "maybe": None if n % 3 == 0 else f"m{n}",
+ }
+ if drift:
+ row["mixed"] = {"k": n} if shard % 2 == 0 else f'{{"k": {n}}}'
+ row["count"] = n if shard % 3 else float(n)
+ row["all_null"] = None if shard % 2 == 0 else f"v{n}"
+ if shard % 4 == 1:
+ del row["maybe"]
+ return row
+
+
+def _write_dataset(base: Path, shards: int, rows: int, drift: bool) -> Path:
+ parquet_dir = base / "parquet-files"
+ parquet_dir.mkdir(parents = True)
+ per = rows // shards
+ for shard in range(shards):
+ frame = pd.DataFrame([_row(shard * per + i, shard, drift) for i in range(per)])
+ frame.to_parquet(parquet_dir / f"batch_{shard:05d}.parquet", index = False)
+ return parquet_dir
+
+
+def _reference(parquet_dir: Path, scratch: Path) -> Path:
+ """What the merge wrote before streaming: the whole folder as a pyarrow-backed dataframe, written back."""
+ copy = scratch / "reference-input"
+ shutil.copytree(parquet_dir, copy)
+ out = scratch / "reference.parquet"
+ _dataframe_read(copy).to_parquet(out, index = False)
+ return out
+
+
+def _assert_same_file(actual: Path, expected: Path) -> None:
+ actual_schema, expected_schema = pq.read_schema(actual), pq.read_schema(expected)
+ assert actual_schema.equals(expected_schema, check_metadata = True)
+ assert str(actual_schema) == str(expected_schema)
+ actual_table, expected_table = pq.read_table(actual), pq.read_table(expected)
+ assert actual_table.equals(expected_table)
+ assert actual_table.to_pylist() == expected_table.to_pylist()
+ assert pq.ParquetFile(actual).metadata.num_row_groups == (
+ pq.ParquetFile(expected).metadata.num_row_groups
+ )
+ assert _dataframe_read(actual).equals(_dataframe_read(expected))
+
+
[email protected]("shards, rows", [(2, 10), (10, 200), (100, 1000)])
[email protected]("drift", [False, True], ids = ["uniform", "drifting"])
+def test_streamed_merge_writes_the_dataframe_merge_file(
+ tmp_path: Path, worker, monkeypatch, shards, rows, drift
+) -> None:
+ base = tmp_path / "dataset"
+ parquet_dir = _write_dataset(base, shards, rows, drift)
+ worker._unify_batch_schemas(base)
+ expected = _reference(parquet_dir, tmp_path)
+
+ def no_dataframe(_path):
+ raise AssertionError("the merge loaded the dataset as a dataframe")
+
+ monkeypatch.setattr(
+ sys.modules["data_designer.config.utils.io_helpers"], "read_parquet_dataset", no_dataframe
+ )
+ worker._merge_batches_to_single_parquet(base)
+
+ assert sorted(p.name for p in parquet_dir.iterdir()) == ["batch_00000.parquet"]
+ assert not list(base.glob("*.merging"))
+ _assert_same_file(parquet_dir / "batch_00000.parquet", expected)
+
+
+def test_streamed_merge_buffers_a_bounded_amount(tmp_path: Path, worker, monkeypatch) -> None:
+ base = tmp_path / "dataset"
+ parquet_dir = _write_dataset(base, 10, 100, drift = False)
+ expected = pq.read_table(_reference(parquet_dir, tmp_path))
+
+ written: list[int] = []
+ original = pq.ParquetWriter.write_table
+
+ def counting(
+ self,
+ table,
+ row_group_size = None,
+ ):
+ written.append(table.num_rows)
+ return original(self, table, row_group_size = row_group_size)
+
+ monkeypatch.setattr(pq.ParquetWriter, "write_table", counting)
+ monkeypatch.setattr(worker, "_MERGE_BUFFER_BYTES", 1)
+ worker._merge_batches_to_single_parquet(base)
+
+ # The zero-row writes are the schema probe.
+ assert [rows for rows in written if rows] == [10] * 10
+ merged = pq.read_table(parquet_dir / "batch_00000.parquet")
+ assert merged.equals(expected)
+
+
+def test_streamed_merge_cuts_row_groups_like_write_table(
+ tmp_path: Path, worker, monkeypatch
+) -> None:
+ base = tmp_path / "dataset"
+ parquet_dir = _write_dataset(base, 10, 50, drift = False)
+ expected = pq.read_table(_reference(parquet_dir, tmp_path))
+ monkeypatch.setattr(worker, "_MERGE_ROW_GROUP_ROWS", 7)
+ worker._merge_batches_to_single_parquet(base)
+
+ metadata = pq.ParquetFile(parquet_dir / "batch_00000.parquet").metadata
+ assert [metadata.row_group(i).num_rows for i in range(metadata.num_row_groups)] == [7] * 7 + [1]
+ assert pq.read_table(parquet_dir / "batch_00000.parquet").equals(expected)
+
+
+def test_shards_whose_types_differ_take_the_dataframe_merge(tmp_path: Path, worker) -> None:
+ # Unify could not settle the types (it logs and keeps them): the dataframe path still decides the merge.
+ base = tmp_path / "dataset"
+ parquet_dir = base / "parquet-files"
+ parquet_dir.mkdir(parents = True)
+ pd.DataFrame({"n": [1, 2]}).to_parquet(parquet_dir / "batch_00000.parquet", index = False)
+ pd.DataFrame({"n": [1.5]}).to_parquet(parquet_dir / "batch_00001.parquet", index = False)
+ calls = []
+ io_helpers = sys.modules["data_designer.config.utils.io_helpers"]
+ real = io_helpers.read_parquet_dataset
+
+ def recording(path):
+ calls.append(path)
+ return pd.concat(
+ [_dataframe_read(p) for p in sorted(Path(path).glob("*.parquet"))], ignore_index = True
+ )
+
+ io_helpers.read_parquet_dataset = recording
+ try:
+ worker._merge_batches_to_single_parquet(base)
+ finally:
+ io_helpers.read_parquet_dataset = real
+
+ assert calls == [parquet_dir]
+ assert pq.read_table(parquet_dir / "batch_00000.parquet").column("n").to_pylist() == [1, 2, 1.5]
+
+
+def test_a_stray_file_the_folder_read_would_pick_up_takes_the_dataframe_merge(
+ tmp_path: Path, worker
+) -> None:
+ base = tmp_path / "dataset"
+ parquet_dir = _write_dataset(base, 2, 4, drift = False)
+ stray = parquet_dir / "batch_00002.parquet.unifying"
+ pq.write_table(pq.read_table(parquet_dir / "batch_00001.parquet"), stray)
+ expected = pq.read_table(_reference(parquet_dir, tmp_path))
+ worker._merge_batches_to_single_parquet(base)
+
+ assert pq.read_table(parquet_dir / "batch_00000.parquet").equals(expected)
+ assert expected.num_rows == 6
+
+
[email protected]("name", ["_partial.parquet", ".partial.parquet"])
+def test_a_shard_name_the_folder_read_skips_stays_out_of_the_merge(
+ tmp_path: Path, worker, name
+) -> None:
+ # The glob lists "_" and "." names, the folder read skips them: the merge must not take their rows.
+ base = tmp_path / "dataset"
+ parquet_dir = _write_dataset(base, 2, 4, drift = False)
+ pq.write_table(pq.read_table(parquet_dir / "batch_00001.parquet"), parquet_dir / name)
+ expected = pq.read_table(_reference(parquet_dir, tmp_path))
+ worker._merge_batches_to_single_parquet(base)
+
+ assert expected.num_rows == 4
+ assert pq.read_table(parquet_dir / "batch_00000.parquet").equals(expected)
+
+
+def test_an_unreadable_shard_fails_the_merge_as_the_folder_read_does(
+ tmp_path: Path, worker
+) -> None:
+ base = tmp_path / "dataset"
+ parquet_dir = _write_dataset(base, 2, 4, drift = False)
+ (parquet_dir / "batch_00001.parquet").write_bytes(b"PAR1 not a parquet file")
+ with pytest.raises(pyarrow.ArrowInvalid, match = "batch_00001.parquet"):
+ worker._merge_batches_to_single_parquet(base)
+ assert sorted(p.name for p in parquet_dir.iterdir()) == [
+ "batch_00000.parquet",
+ "batch_00001.parquet",
+ ]
+ assert not list(base.glob("*.merging"))
+
+
+def test_a_failed_background_write_fails_the_merge_and_keeps_the_shards(
+ tmp_path: Path, worker, monkeypatch
+) -> None:
+ base = tmp_path / "dataset"
+ parquet_dir = _write_dataset(base, 10, 100, drift = False)
+ before = {p.name: p.read_bytes() for p in parquet_dir.iterdir()}
+ original = pq.ParquetWriter.write_table
+ calls: list[int] = []
+
+ def failing(
+ self,
+ table,
+ row_group_size = None,
+ ):
+ if table.num_rows:
+ calls.append(table.num_rows)
+ if len(calls) == 3:
+ raise OSError("disk full")
+ return original(self, table, row_group_size = row_group_size)
+
+ monkeypatch.setattr(pq.ParquetWriter, "write_table", failing)
+ monkeypatch.setattr(worker, "_MERGE_BUFFER_BYTES", 1)
+ with pytest.raises(OSError, match = "disk full"):
+ worker._merge_batches_to_single_parquet(base)
+
+ assert {p.name: p.read_bytes() for p in parquet_dir.iterdir()} == before
+ assert not list(base.glob("*.merging"))