Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep Data Recipe runs whose structured fields change type between rows (#13407)
Daniel Han4e49365d6028
diff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.pyindex 2d2d4e06ab6..4d9283d94bc 100644--- a/studio/backend/core/data_recipe/jobs/worker.py+++ b/studio/backend/core/data_recipe/jobs/worker.py@@ -17,7 +17,13 @@ from typing import Anyfrom ..jsonable import to_jsonable, to_preview_jsonable_rowfrom .constants import EVENT_JOB_COMPLETED, EVENT_JOB_ERROR, EVENT_JOB_STARTED-from ..service import build_config_builder, create_data_designer, install_public_egress_guard+from ..service import (+ build_config_builder,+ create_data_designer,+ _unify_cell_types,+ install_public_egress_guard,+ write_parquet_with_unified_types,+)from utils.paths.lazy import LazyPathfrom utils.paths import ensure_dir, recipe_datasets_root@@ -181,6 +187,7 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])else:results = designer.create(builder, num_records = rows, dataset_name = dataset_name)analysis = to_jsonable(results.load_analysis().model_dump(mode = "json"))+ _unify_batch_schemas(results.artifact_storage.base_dataset_path)if merge_batches:_merge_batches_to_single_parquet(results.artifact_storage.base_dataset_path)artifact_path = str(results.artifact_storage.base_dataset_path)@@ -204,6 +211,89 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any]))+def _unify_batch_schemas(base_dataset_path: Path) -> None:+ """Give every batch file one schema. Each batch infers its own, so a field stored as JSON text in one+ batch and as an object in another (or float vs int, all-null vs text) left a folder that training+ could not load as one dataset."""+ parquet_dir = base_dataset_path / "parquet-files"+ parquet_files = drop_appledouble_metadata(sorted(parquet_dir.glob("*.parquet")))+ if len(parquet_files) <= 1:+ return++ import pyarrow+ import pyarrow.parquet as pyarrow_parquet++ schemas = [pyarrow_parquet.read_schema(path).remove_metadata() for path in parquet_files]+ if all(schema.equals(schemas[0]) for schema in schemas[1:]):+ return++ staged = [path.with_name(path.name + ".unifying") for path in parquet_files]+ try:+ names = list(dict.fromkeys(name for schema in schemas for name in schema.names))+ # Only differing columns change. Arrow-promotable ones are cast per file while writing; struct vs text+ # needs every row at once (which fields turn into JSON text depends on all of them), so only those are held.+ targets: dict[str, Any] = {}+ rebuilt: dict[str, list[Any]] = {}+ for name in names:+ fields = [schema.field(name) for schema in schemas if name in schema.names]+ if len(fields) == len(schemas) and all(f.type == fields[0].type for f in fields):+ continue+ try:+ targets[name] = (+ pyarrow.unify_schemas(+ [pyarrow.schema([field]) for field in fields],+ promote_options = "permissive",+ )+ .field(name)+ .type+ )+ continue+ except (pyarrow.ArrowInvalid, pyarrow.ArrowTypeError, pyarrow.ArrowNotImplementedError):+ pass+ columns = [+ pyarrow_parquet.read_table(path, columns = [name]).column(name)+ if name in schema.names+ else pyarrow.nulls(pyarrow_parquet.read_metadata(path).num_rows)+ for path, schema in zip(parquet_files, schemas)+ ]+ values = _unify_cell_types([v for column in columns for v in column.to_pylist()])+ merged = pyarrow.array(values, from_pandas = True)+ del values+ rebuilt[name], offset = [], 0+ for column in columns:+ rebuilt[name].append(merged.slice(offset, len(column)))+ offset += len(column)+ for index, (path, target_path) in enumerate(zip(parquet_files, staged)):+ table = pyarrow_parquet.read_table(path)+ for name in names:+ if name in rebuilt:+ column = rebuilt[name][index]+ elif name in targets:+ column = (+ table.column(name)+ if name in table.column_names+ else pyarrow.nulls(table.num_rows)+ ).cast(targets[name])+ else:+ continue+ if name in table.column_names:+ table = table.set_column(table.column_names.index(name), name, column)+ else:+ table = table.append_column(name, column)+ pyarrow_parquet.write_table(+ table.select(names).replace_schema_metadata(None), target_path+ )+ except Exception as exc:+ for path in staged:+ path.unlink(missing_ok = True)+ logging.getLogger("data_designer.unsloth").warning(+ "Batch files kept their own column types: %s", exc+ )+ return+ for path, final in zip(staged, parquet_files):+ path.replace(final)++def _merge_batches_to_single_parquet(base_dataset_path: Path) -> None:parquet_dir = base_dataset_path / "parquet-files"# Counted, so a companion would make a single-batch job take the merge path and rewrite a dataset that needed none.@@ -217,10 +307,21 @@ def _merge_batches_to_single_parquet(base_dataset_path: Path) -> None:returndataframe = read_parquet_dataset(parquet_dir)+ # Written beside the shards first: a failed write must not take the generated batches with it.+ staged_file = base_dataset_path / "batch_00000.parquet.merging"+ try:+ write_parquet_with_unified_types(+ dataframe,+ staged_file,+ lambda frame: frame.to_parquet(staged_file, index = False),+ )+ except BaseException:+ staged_file.unlink(missing_ok = True)+ raiseshutil.rmtree(parquet_dir)parquet_dir.mkdir(parents = True, exist_ok = True)merged_file = parquet_dir / "batch_00000.parquet"- dataframe.to_parquet(merged_file, index = False)+ staged_file.replace(merged_file)_rewrite_merged_metadata(base_dataset_path = base_dataset_path,parquet_file = merged_file,diff --git a/studio/backend/core/data_recipe/service.py b/studio/backend/core/data_recipe/service.pyindex e73fff0bce2..8e53f5b09d9 100644--- a/studio/backend/core/data_recipe/service.py+++ b/studio/backend/core/data_recipe/service.py@@ -7,6 +7,8 @@ from core.training.account_jobs import account_path, managed_account, validate_rimport base64import functoolsimport io+import json+import loggingimport mathimport osfrom pathlib import Path@@ -16,7 +18,7 @@ from fastapi import HTTPExceptionfrom utils.paths import recipe_datasets_root-from .jsonable import to_jsonable+from .jsonable import _is_pandas_missing, to_jsonablefrom .local_callable_validators import (register_oxc_local_callable_validators,split_oxc_local_callable_validators,@@ -250,6 +252,151 @@ def _apply_data_designer_expression_blank_patch() -> None:setattr(ExpressionColumnGenerator, "_unsloth_expression_blank_patch_applied", True)+def _is_missing_cell(value: Any) -> bool:+ return (+ value is None+ or (isinstance(value, float) and math.isnan(value))+ or _is_pandas_missing(value)+ )+++def _as_json_text(value: Any) -> Any:+ if _is_missing_cell(value):+ return None+ if isinstance(value, str):+ return value+ value = to_jsonable(value)+ if value is None or isinstance(value, str):+ return value+ return json.dumps(value, ensure_ascii = False, default = str)+++def _unify_cell_types(values: list[Any]) -> list[Any]:+ """Make one path of a column Arrow-writable: dicts and lists recurse, and a path whose rows disagree+ (text in one row, an object in another) is stored as JSON text instead of failing the batch."""+ import pyarrow++ present = [value for value in values if not _is_missing_cell(value)]+ if not present:+ return values+ if all(isinstance(value, dict) for value in present):+ keys = list(dict.fromkeys(key for value in present for key in value))+ if keys and all(isinstance(key, str) for key in keys):+ fields = {+ key: _unify_cell_types(+ [value.get(key) if isinstance(value, dict) else None for value in values]+ )+ for key in keys+ }+ return [+ {key: fields[key][index] for key in value} if isinstance(value, dict) else value+ for index, value in enumerate(values)+ ]+ elif all(isinstance(value, (list, tuple)) for value in present):+ flat = _unify_cell_types([item for value in present for item in value])+ result, offset = [], 0+ for value in values:+ if isinstance(value, (list, tuple)):+ result.append(flat[offset : offset + len(value)])+ offset += len(value)+ else:+ result.append(value)+ return result+ elif not any(isinstance(value, (dict, list, tuple)) for value in present):+ kinds = {type(value) for value in present}+ # Arrow infers a mixed path from its first value (a date then a datetime drops the time); int/float is safe.+ if len(kinds) > 1 and not kinds <= {int, float}:+ return [_as_json_text(value) for value in values]+ try:+ pyarrow.array(present, from_pandas = True)+ return values+ except (pyarrow.ArrowInvalid, pyarrow.ArrowTypeError):+ pass+ return [_as_json_text(value) for value in values]+++def _unify_dataframe_cell_types(dataframe: Any) -> tuple[Any, list[str]]:+ import pyarrow++ unified = dataframe.copy()+ changed: list[str] = []+ for column in unified.columns:+ if unified[column].dtype != object:+ continue+ values = unified[column].tolist()+ try:+ pyarrow.array(values, from_pandas = True)+ continue+ except (pyarrow.ArrowInvalid, pyarrow.ArrowTypeError):+ pass+ unified[column] = _unify_cell_types(values)+ changed.append(str(column))+ return unified, changed+++_MIXED_TYPE_WARNED: set[tuple[str, ...]] = set()+++def write_parquet_with_unified_types(dataframe: Any, path: Any, write: Any) -> Any:+ """Run ``write(dataframe)``; if Arrow refuses a column whose rows hold different types at the same path+ (unslothai/unsloth#5848: an LLM Structured field returned as text in one row and an object in another),+ store those paths as JSON text and write again."""+ import pyarrow+ try:+ return write(dataframe)+ except (pyarrow.ArrowInvalid, pyarrow.ArrowTypeError) as exc:+ unified, changed = _unify_dataframe_cell_types(dataframe)+ if not changed:+ raise+ try:+ result = write(unified)+ except (pyarrow.ArrowInvalid, pyarrow.ArrowTypeError):+ raise exc from None+ # data-designer rewrites a batch once per column stage; say it once.+ key = (str(path), *changed)+ if key not in _MIXED_TYPE_WARNED:+ _MIXED_TYPE_WARNED.add(key)+ logging.getLogger("data_designer.unsloth").warning(+ "Column(s) %s returned different value types across rows; mismatched fields were "+ "saved as JSON text.",+ ", ".join(repr(name) for name in changed),+ )+ return result+++def _apply_data_designer_mixed_type_parquet_patch() -> None:+ try:+ from data_designer.engine.storage.artifact_storage import ( # pyright: ignore[reportMissingImports]+ ArtifactStorage,+ )+ except ImportError:+ return++ if getattr(ArtifactStorage, "_unsloth_mixed_type_parquet_patch_applied", False):+ return++ original_write = ArtifactStorage.write_parquet_file++ @functools.wraps(original_write)+ def _patched_write(+ self: Any,+ parquet_file_name: str,+ dataframe: Any,+ batch_stage: Any,+ subfolder: str | None = None,+ ) -> Any:+ return write_parquet_with_unified_types(+ dataframe,+ Path(str(getattr(self, "base_dataset_path", ""))) / parquet_file_name,+ lambda frame: original_write(+ self, parquet_file_name, frame, batch_stage, subfolder = subfolder+ ),+ )++ ArtifactStorage.write_parquet_file = _patched_write+ setattr(ArtifactStorage, "_unsloth_mixed_type_parquet_patch_applied", True)++def _require_public_provider_endpoint(endpoint: str) -> None:"""The recipe engine dials providers itself, so a managed account's endpoint cannot use the pinnedtransport: require HTTPS, which binds the peer to its certificate rather than to a DNS answer that@@ -522,6 +669,7 @@ def create_data_designer(recipe: dict[str, Any], *, artifact_path: str | None =_apply_data_designer_image_context_patch()_apply_data_designer_prompt_blank_patch()_apply_data_designer_expression_blank_patch()+ _apply_data_designer_mixed_type_parquet_patch()from data_designer.interface.data_designer import DataDesigner # pyright: ignore[reportMissingImports]if artifact_path is None:diff --git a/studio/backend/tests/test_data_recipe_mixed_type_parquet.py b/studio/backend/tests/test_data_recipe_mixed_type_parquet.pynew file mode 100644index 00000000000..9d995d77d09--- /dev/null+++ b/studio/backend/tests/test_data_recipe_mixed_type_parquet.py@@ -0,0 +1,289 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import json+import threading+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from pathlib import Path++import pytest++pytest.importorskip("pyarrow")++import pandas as pd+import pyarrow+import pyarrow.parquet as pq++TEXT_ROW = {"instruction": "Explain caching", "output": "Caches keep responses."}+OBJECT_ROW = {"instruction": "Give a config", "output": {"debug": True, "level": "info"}}++[email protected](name = "provider")+def fixture_provider():+ lock = threading.Lock()+ calls = [0]++ class Handler(BaseHTTPRequestHandler):+ def do_POST(self):+ self.rfile.read(int(self.headers["Content-Length"]))+ with lock:+ calls[0] += 1+ answer = TEXT_ROW if calls[0] % 2 else OBJECT_ROW+ content = "```json\n" + json.dumps(answer) + "\n```"+ reply = json.dumps(+ {+ "id": "c",+ "object": "chat.completion",+ "created": 0,+ "model": "x",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": content},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},+ }+ ).encode()+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(reply)))+ self.end_headers()+ self.wfile.write(reply)++ def log_message(self, *args):+ pass++ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ threading.Thread(target = server.serve_forever, daemon = True).start()+ yield f"http://127.0.0.1:{server.server_address[1]}/v1"+ server.shutdown()+++def test_full_run_keeps_a_structured_field_whose_type_varies_by_row(+ tmp_path: Path, provider+) -> None:+ # unslothai/unsloth#5848: the Output Format box takes a JSON shape, so nothing pins a field's type; one row+ # answering with text and another with an object failed the whole run after every LLM call was paid for.+ pytest.importorskip("data_designer")+ from core.data_recipe.service import build_config_builder, create_data_designer++ recipe = {+ "model_providers": [{"name": "p", "endpoint": provider, "provider_type": "openai"}],+ "model_configs": [+ {+ "alias": "m",+ "model": "x",+ "provider": "p",+ "inference_parameters": {"max_parallel_requests": 1},+ }+ ],+ "columns": [+ {+ "column_type": "sampler",+ "name": "topic",+ "sampler_type": "category",+ "params": {"values": ["web", "db"]},+ },+ {+ "column_type": "llm-structured",+ "name": "dataset_row",+ "model_alias": "m",+ "prompt": "One pair about {{ topic }}.",+ "output_format": {"instruction": "string", "output": "string"},+ },+ ],+ }+ designer = create_data_designer(recipe, artifact_path = str(tmp_path / "artifacts"))+ results = designer.create(build_config_builder(recipe), num_records = 4, dataset_name = "run")++ rows = pq.read_table(results.artifact_storage.final_dataset_path).to_pylist()+ outputs = sorted(row["dataset_row"]["output"] for row in rows)+ assert len(rows) == 4+ assert {row["dataset_row"]["instruction"] for row in rows} == {+ TEXT_ROW["instruction"],+ OBJECT_ROW["instruction"],+ }+ assert TEXT_ROW["output"] in outputs+ assert {"debug": True, "level": "info"} in [json.loads(o) for o in outputs if o.startswith("{")]+++def test_unify_only_rewrites_the_disagreeing_paths() -> None:+ from core.data_recipe.service import _unify_cell_types++ values = [+ {"messages": [{"role": "user", "content": "hi"}], "score": 1},+ {+ "messages": [{"role": "user", "content": [{"type": "text", "text": "yo"}]}],+ "score": "high",+ },+ None,+ ]+ unified = _unify_cell_types(values)+ pyarrow.array(unified, from_pandas = True)+ assert unified[0] == {"messages": [{"role": "user", "content": "hi"}], "score": "1"}+ assert unified[1]["messages"][0]["content"] == '[{"type": "text", "text": "yo"}]'+ assert unified[1]["score"] == "high"+ assert unified[2] is None+++def test_writable_frames_are_written_untouched(tmp_path: Path) -> None:+ from core.data_recipe.service import write_parquet_with_unified_types++ frame = pd.DataFrame({"row": [TEXT_ROW, dict(TEXT_ROW)]})+ seen = []+ write_parquet_with_unified_types(frame, tmp_path / "x.parquet", seen.append)+ assert len(seen) == 1 and seen[0] is frame+++def test_write_that_still_fails_raises_the_original_error(tmp_path: Path) -> None:+ from core.data_recipe.service import write_parquet_with_unified_types++ frame = pd.DataFrame({"row": [TEXT_ROW, OBJECT_ROW]})++ def write(_frame):+ raise pyarrow.ArrowInvalid("still broken" if _frame is not frame else "original")++ with pytest.raises(pyarrow.ArrowInvalid, match = "original"):+ write_parquet_with_unified_types(frame, tmp_path / "x.parquet", write)+++def _write_shards(base: Path) -> Path:+ parquet_dir = base / "parquet-files"+ parquet_dir.mkdir(parents = True)+ pq.write_table(+ pyarrow.Table.from_pylist([{"row": TEXT_ROW}]), parquet_dir / "batch_00000.parquet"+ )+ pq.write_table(+ pyarrow.Table.from_pylist([{"row": OBJECT_ROW}]), parquet_dir / "batch_00001.parquet"+ )+ return parquet_dir+++def test_merge_joins_batches_whose_field_types_differ(tmp_path: Path) -> None:+ pytest.importorskip("data_designer")+ from core.data_recipe.jobs.worker import _merge_batches_to_single_parquet++ parquet_dir = _write_shards(tmp_path)+ _merge_batches_to_single_parquet(tmp_path)++ assert sorted(p.name for p in parquet_dir.iterdir()) == ["batch_00000.parquet"]+ rows = pq.read_table(parquet_dir / "batch_00000.parquet").to_pylist()+ assert [row["row"]["output"] for row in rows] == [+ TEXT_ROW["output"],+ '{"debug": true, "level": "info"}',+ ]+++def test_failed_merge_keeps_the_generated_batches(tmp_path: Path, monkeypatch) -> None:+ pytest.importorskip("data_designer")+ from core.data_recipe.jobs import worker++ parquet_dir = _write_shards(tmp_path)++ def fail(*_args, **_kwargs):+ raise OSError("disk full")++ monkeypatch.setattr(worker, "write_parquet_with_unified_types", fail)+ with pytest.raises(OSError):+ worker._merge_batches_to_single_parquet(tmp_path)++ assert sorted(p.name for p in parquet_dir.iterdir()) == [+ "batch_00000.parquet",+ "batch_00001.parquet",+ ]+ assert not list(tmp_path.glob("*.merging"))+++ "first, second, expected",+ [+ ({"o": {"d": True}}, {"o": "x"}, [{"o": '{"d": true}'}, {"o": "x"}]),+ (1, 1.5, [1.0, 1.5]),+ (None, "s", [None, "s"]),+ ({"a": "x"}, {"a": "y", "b": "z"}, [{"a": "x", "b": None}, {"a": "y", "b": "z"}]),+ ],+)+def test_batches_with_different_schemas_load_as_one_dataset(+ tmp_path: Path, first, second, expected+) -> None:+ from core.data_recipe.jobs.worker import _unify_batch_schemas++ parquet_dir = tmp_path / "parquet-files"+ parquet_dir.mkdir()+ pd.DataFrame({"r": [first], "n": [0]}).to_parquet(+ parquet_dir / "batch_00000.parquet", index = False+ )+ pd.DataFrame({"r": [second], "n": [1]}).to_parquet(+ parquet_dir / "batch_00001.parquet", index = False+ )++ _unify_batch_schemas(tmp_path)++ assert sorted(p.name for p in parquet_dir.iterdir()) == [+ "batch_00000.parquet",+ "batch_00001.parquet",+ ]+ table = pq.read_table(parquet_dir)+ assert table.column("r").to_pylist() == expected+ assert table.column("n").to_pylist() == [0, 1]+++def test_batches_that_already_agree_are_not_rewritten(tmp_path: Path) -> None:+ from core.data_recipe.jobs.worker import _unify_batch_schemas++ parquet_dir = _write_shards(tmp_path)+ for path in parquet_dir.iterdir():+ path.unlink()+ pd.DataFrame({"r": ["a"]}).to_parquet(parquet_dir / "batch_00000.parquet", index = False)+ pd.DataFrame({"r": ["b"]}).to_parquet(parquet_dir / "batch_00001.parquet", index = False)+ before = {p.name: p.read_bytes() for p in parquet_dir.iterdir()}++ _unify_batch_schemas(tmp_path)++ assert {p.name: p.read_bytes() for p in parquet_dir.iterdir()} == before+++def test_pandas_missing_values_do_not_flatten_the_struct() -> None:+ from core.data_recipe.service import _unify_cell_types++ unified = _unify_cell_types([{"o": {"d": True}}, {"o": "x"}, pd.NA])+ pyarrow.array(unified, from_pandas = True)+ assert unified[:2] == [{"o": '{"d": true}'}, {"o": "x"}]+++def test_batch_unify_leaves_agreeing_columns_and_promotes_with_arrow(tmp_path: Path) -> None:+ from core.data_recipe.jobs.worker import _unify_batch_schemas++ parquet_dir = tmp_path / "parquet-files"+ parquet_dir.mkdir()+ stamp = pd.Timestamp("2026-01-01 12:34:56.123456789")+ for index, (row, when) in enumerate([({"o": {"d": True}}, None), ({"o": "x"}, stamp)]):+ table = pyarrow.table(+ {+ "r": pyarrow.array([row]),+ "ts": pyarrow.array([stamp], pyarrow.timestamp("ns")),+ "maybe": pyarrow.array([when], pyarrow.timestamp("ns") if when else pyarrow.null()),+ }+ )+ pq.write_table(table, parquet_dir / f"batch_{index:05d}.parquet")++ _unify_batch_schemas(tmp_path)++ table = pq.read_table(parquet_dir)+ assert table.schema.field("ts").type == pyarrow.timestamp("ns")+ assert table.schema.field("maybe").type == pyarrow.timestamp("ns")+ assert table.column("ts").to_pylist() == [stamp, stamp]+ assert table.column("maybe").to_pylist() == [None, stamp]+ assert table.column("r").to_pylist() == [{"o": '{"d": true}'}, {"o": "x"}]+++def test_dates_and_datetimes_on_one_path_keep_their_time() -> None:+ import datetime++ from core.data_recipe.service import _unify_cell_types++ values = [datetime.date(2026, 1, 2), datetime.datetime(2026, 1, 2, 12, 34, 56, 123456), None]+ assert _unify_cell_types(values) == ["2026-01-02", "2026-01-02T12:34:56.123456", None]+ assert _unify_cell_types([1, 2.5]) == [1, 2.5]