Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Export recipe dataset artifacts to downloadable files."""
from __future__ import annotations
import base64
import json
import math
import tempfile
import zipfile
from decimal import Decimal
from pathlib import Path
from typing import Any, Literal
from core.data_recipe.huggingface import (
RecipeDatasetPublishError,
_resolve_recipe_artifact_path,
)
from core.data_recipe.jsonable import to_jsonable
from utils.paths.path_utils import drop_appledouble_metadata
ExportFormat = Literal["jsonl", "parquet"]
_JSONL_EXPORT_BATCH_ROWS = 8192
class RecipeDatasetExportError(ValueError):
"""Raised when a recipe dataset cannot be exported."""
class _JsonNumber(str):
"""A validated JSON number that must be emitted without string quotes."""
def _parquet_dir(dataset_path: Path) -> Path:
parquet_dir = dataset_path / "parquet-files"
if not parquet_dir.exists():
raise RecipeDatasetExportError(f"Dataset parquet files missing: {parquet_dir}")
if not _parquet_files(parquet_dir):
raise RecipeDatasetExportError(f"No parquet files found in {parquet_dir}")
return parquet_dir
def _parquet_files(parquet_dir: Path) -> list[Path]:
"""The real shards, without the ``._batch.parquet`` companions a macOS volume leaves beside
them: the glob matches those but no reader can parse one. Same call the worker makes."""
return drop_appledouble_metadata(sorted(parquet_dir.glob("*.parquet")))
def _sanitize_json_value(value: Any) -> Any:
if isinstance(value, float):
return None if not math.isfinite(value) else value
if isinstance(value, Decimal):
return None if not value.is_finite() else _JsonNumber(str(value))
if isinstance(value, (bytes, bytearray)):
return base64.b64encode(bytes(value)).decode("ascii")
if isinstance(value, dict):
return {str(key): _sanitize_json_value(item) for key, item in value.items()}
if isinstance(value, (list, tuple, set)):
return [_sanitize_json_value(item) for item in value]
converted = to_jsonable(value)
if converted is not value:
return _sanitize_json_value(converted)
if isinstance(converted, float):
return None if not math.isfinite(converted) else converted
if converted is None or isinstance(converted, (str, int, bool)):
return converted
return str(converted)
def _json_dumps_value(value: Any) -> str:
if isinstance(value, _JsonNumber):
return str(value)
if isinstance(value, dict):
return (
"{"
+ ", ".join(
f"{json.dumps(str(key), ensure_ascii = False)}: {_json_dumps_value(item)}"
for key, item in value.items()
)
+ "}"
)
if isinstance(value, list):
return "[" + ", ".join(_json_dumps_value(item) for item in value) + "]"
return json.dumps(value, ensure_ascii = False, allow_nan = False)
# Deeper rows take the general path, which owns how a too-deep value fails.
_PLAIN_JSON_MAX_DEPTH = 32
class _TooDeep(Exception):
pass
def _is_plain_json(value: Any, depth: int = 0) -> bool:
"""Exact str / int / bool / None / finite float leaves under str-keyed dicts and lists, which
one ``json.dumps`` writes byte-for-byte as the sanitize + per-value path does."""
value_type = type(value)
if value_type is str or value_type is int or value_type is bool or value is None:
return True
if value_type is float:
return math.isfinite(value)
if depth >= _PLAIN_JSON_MAX_DEPTH:
raise _TooDeep
if value_type is dict:
for key, item in value.items():
if type(key) is not str or not _is_plain_json(item, depth + 1):
return False
return True
if value_type is list:
for item in value:
if not _is_plain_json(item, depth + 1):
return False
return True
return False
def _check_depth(value: Any, depth: int) -> None:
if isinstance(value, (dict, list, tuple, set)):
if depth >= _PLAIN_JSON_MAX_DEPTH:
raise _TooDeep
for item in value.values() if isinstance(value, dict) else value:
_check_depth(item, depth + 1)
def _json_dumps_row(row: dict[str, Any]) -> str:
if type(row) is dict:
try:
if _is_plain_json(row):
return json.dumps(row, ensure_ascii = False, allow_nan = False)
# Non-str keys can collide under str(key): only str-keyed rows go column by column.
if all(type(key) is str for key in row):
# Sanitize every column before writing any so a failing row raises the same error.
columns = []
all_plain = True
for key, item in row.items():
plain = _is_plain_json(item, 1)
if not plain:
_check_depth(item, 1)
item = _sanitize_json_value(item)
plain = _is_plain_json(item, 1)
all_plain = all_plain and plain
columns.append((key, item, plain))
if all_plain:
return json.dumps(
{key: item for key, item, _ in columns},
ensure_ascii = False,
allow_nan = False,
)
return (
"{"
+ ", ".join(
f"{json.dumps(key, ensure_ascii = False)}: "
+ (
json.dumps(item, ensure_ascii = False, allow_nan = False)
if plain
else _json_dumps_value(item)
)
for key, item, plain in columns
)
+ "}"
)
except _TooDeep:
pass
return _json_dumps_value(_sanitize_json_value(row))
def write_jsonl_rows(handle, rows: list[dict[str, Any]]) -> None:
"""Write JSONL rows with the same durable value encoding used for parquet artifacts."""
for row in rows:
handle.write(_json_dumps_row(row))
handle.write("\n")
def _write_jsonl_from_parquet(parquet_dir: Path, destination: Path) -> None:
"""Stream sorted parquet shards through the one supported JSONL reader."""
try:
import pyarrow.parquet as pyarrow_parquet # type: ignore
except ImportError as exc:
raise RecipeDatasetExportError("PyArrow is required to export recipe datasets.") from exc
parquet_files = _parquet_files(parquet_dir)
try:
with destination.open("w", encoding = "utf-8") as handle:
for path in parquet_files:
parquet_file = pyarrow_parquet.ParquetFile(path)
for batch in parquet_file.iter_batches(batch_size = _JSONL_EXPORT_BATCH_ROWS):
write_jsonl_rows(
handle,
batch.to_pylist(),
)
except Exception as exc:
raise RecipeDatasetExportError(f"Could not export recipe parquet data: {exc}") from exc
def safe_filename_stem(value: str) -> str:
cleaned = "".join(ch if ch.isalnum() or ch in {"-", "_"} else "-" for ch in value.strip())
cleaned = cleaned.strip("-_")
return cleaned or "recipe-dataset"
def _artifact_image_files(dataset_path: Path) -> list[Path]:
images_dir = dataset_path / "images"
if not images_dir.is_dir():
return []
return [
path for path in drop_appledouble_metadata(sorted(images_dir.rglob("*"))) if path.is_file()
]
def _processor_parquet_dirs(dataset_path: Path) -> list[Path]:
processors_dir = dataset_path / "processors-files"
if not processors_dir.is_dir():
return []
return [
path
for path in drop_appledouble_metadata(sorted(processors_dir.iterdir()))
if path.is_dir() and _parquet_files(path)
]
def download_filename(*, artifact_path: str, export_format: ExportFormat, stem: str) -> str:
"""The name the export will have, and the checks ``build_dataset_download`` will make, so a
run whose shards are gone is refused while the caller can still be told."""
try:
dataset_path = _resolve_recipe_artifact_path(artifact_path)
except RecipeDatasetPublishError as exc:
raise RecipeDatasetExportError(str(exc)) from exc
_parquet_dir(dataset_path)
if export_format == "parquet":
return f"{stem}.parquet.zip"
if _artifact_image_files(dataset_path) or _processor_parquet_dirs(dataset_path):
return f"{stem}.jsonl.zip"
return f"{stem}.jsonl"
def _add_images_to_archive(archive: zipfile.ZipFile, dataset_path: Path) -> None:
images_dir = dataset_path / "images"
if not images_dir.is_dir():
return
for image_file in drop_appledouble_metadata(sorted(images_dir.rglob("*"))):
if not image_file.is_file():
continue
relative_path = image_file.relative_to(images_dir)
# Stored, not deflated: deflating already-compressed images bought nothing and cost 7x.
archive.write(
image_file,
arcname = str(Path("images") / relative_path),
compress_type = zipfile.ZIP_STORED,
)
def build_dataset_download(
*, artifact_path: str, export_format: ExportFormat, filename_stem: str
) -> tuple[Path, str, str]:
"""Returns (temp_path, media_type, download_filename); the caller deletes temp_path once sent."""
try:
dataset_path = _resolve_recipe_artifact_path(artifact_path)
except RecipeDatasetPublishError as exc:
raise RecipeDatasetExportError(str(exc)) from exc
stem = safe_filename_stem(filename_stem)
parquet_dir = _parquet_dir(dataset_path)
if export_format == "parquet":
tmp = tempfile.NamedTemporaryFile(delete = False, suffix = ".zip")
tmp.close()
zip_path = Path(tmp.name)
# Only a successful build's caller unlinks the temp file, so a failed one takes its own.
try:
with zipfile.ZipFile(zip_path, "w", compression = zipfile.ZIP_DEFLATED) as archive:
for parquet_file in _parquet_files(parquet_dir):
archive.write(parquet_file, arcname = parquet_file.name)
for processor_dir in _processor_parquet_dirs(dataset_path):
for parquet_file in _parquet_files(processor_dir):
archive.write(
parquet_file,
arcname = str(
Path("processors") / processor_dir.name / parquet_file.name
),
)
_add_images_to_archive(archive, dataset_path)
except BaseException:
zip_path.unlink(missing_ok = True)
raise
return zip_path, "application/zip", f"{stem}.parquet.zip"
tmp = tempfile.NamedTemporaryFile(delete = False, suffix = ".jsonl")
tmp.close()
jsonl_path = Path(tmp.name)
try:
_write_jsonl_from_parquet(parquet_dir, jsonl_path)
image_files = _artifact_image_files(dataset_path)
processor_dirs = _processor_parquet_dirs(dataset_path)
if not image_files and not processor_dirs:
return jsonl_path, "application/x-ndjson", f"{stem}.jsonl"
# Rows reference images by relative path, so a bare JSONL loses every image.
zip_tmp = tempfile.NamedTemporaryFile(delete = False, suffix = ".zip")
zip_tmp.close()
zip_path = Path(zip_tmp.name)
try:
with zipfile.ZipFile(zip_path, "w", compression = zipfile.ZIP_DEFLATED) as archive:
archive.write(jsonl_path, arcname = f"{stem}.jsonl")
for processor_dir in processor_dirs:
_write_jsonl_from_parquet(processor_dir, jsonl_path)
archive.write(
jsonl_path,
arcname = str(Path("processors") / f"{processor_dir.name}.jsonl"),
)
_add_images_to_archive(archive, dataset_path)
except BaseException:
zip_path.unlink(missing_ok = True)
raise
except BaseException:
jsonl_path.unlink(missing_ok = True)
raise
jsonl_path.unlink(missing_ok = True)
return zip_path, "application/zip", f"{stem}.jsonl.zip"