Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: live dataset preview during active runs (#10737)
Sourav Rajvi25fabecd6fb3
diff --git a/studio/backend/core/data_recipe/jobs/manager.py b/studio/backend/core/data_recipe/jobs/manager.pyindex ab2850a8109..93cb03eda60 100644--- a/studio/backend/core/data_recipe/jobs/manager.py+++ b/studio/backend/core/data_recipe/jobs/manager.py@@ -359,7 +359,15 @@ class JobManager:base_dataset_path = Path(artifact_path)parquet_dir = base_dataset_path / "parquet-files"if not parquet_dir.exists():- return {"error": f"dataset path missing: {parquet_dir}"}+ if job_status in {"completed", "error", "cancelled"}:+ return {"error": f"dataset path missing: {parquet_dir}"}+ return None+ if job_status not in {"completed", "error", "cancelled"}:+ # DuckDB opens with FILE_SHARE_DELETE; the pyarrow fallback would block the+ # worker's merge rmtree on Windows.+ return self._load_dataset_page_with_duckdb(+ parquet_dir = parquet_dir, limit = limit, offset = offset+ )return self._load_dataset_page(parquet_dir = parquet_dir, limit = limit, offset = offset)except Exception as exc:@@ -393,29 +401,36 @@ class JobManager:try:conn = duckdb.connect(":memory:")try:- total_row = conn.execute(- "SELECT COUNT(*) FROM read_parquet(?)",+ # Row counts from the footers, then scan only the shards the page covers: a full+ # sort per request grows with the run and the live preview polls page 1.+ shards = conn.execute(+ "SELECT file_name, num_rows FROM parquet_file_metadata(?) ORDER BY file_name",[parquet_glob],- ).fetchone()- total = int(total_row[0] if total_row else 0)- dataframe = conn.execute(- (- "SELECT *, row_number() OVER (PARTITION BY filename) AS __row_num__ "- "FROM read_parquet(?, filename=true) "- "ORDER BY filename, __row_num__ "- "LIMIT ? OFFSET ?"- ),- [parquet_glob, int(limit), int(offset)],- ).fetchdf()+ ).fetchall()+ total = sum(int(num_rows) for _, num_rows in shards)+ frames = []+ skip, remaining = int(offset), int(limit)+ for file_name, num_rows in shards:+ if remaining <= 0:+ break+ if skip >= int(num_rows):+ skip -= int(num_rows)+ continue+ frame = conn.execute(+ "SELECT * FROM read_parquet(?) LIMIT ? OFFSET ?",+ [file_name, remaining, skip],+ ).fetchdf()+ frames.append(frame)+ remaining -= len(frame.index)+ skip = 0finally:conn.close()except (RuntimeError, ValueError, duckdb.Error):return None- for helper_col in ("filename", "__row_num__"):- if helper_col in dataframe.columns:- dataframe = dataframe.drop(columns = [helper_col])+ import pandas as pd+ dataframe = pd.concat(frames, ignore_index = True) if frames else pd.DataFrame()rows = dataframe.to_dict(orient = "records")return {"dataset": to_preview_jsonable_row(rows), "total": total}@@ -587,6 +602,8 @@ class JobManager:returnif et == EVENT_JOB_STARTED:self._job.status = "active"+ self._job.artifact_path = event.get("artifact_path") or self._job.artifact_path+ self._job.execution_type = event.get("execution_type") or self._job.execution_typeif et == EVENT_JOB_COMPLETED:self._job.status = "completed"self._job.finished_at = time.time()diff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.pyindex 9b8b04d9d7f..b4c5c4d5bdb 100644--- a/studio/backend/core/data_recipe/jobs/worker.py+++ b/studio/backend/core/data_recipe/jobs/worker.py@@ -97,11 +97,7 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])env = os.getenv("ENVIRONMENT_TYPE", "production"),)- event_queue.put({"type": EVENT_JOB_STARTED, "ts": time.time()})-try:- from data_designer.config.run_config import RunConfig-rows = int(run.get("rows") or 1000)job_id = str(run.get("_job_id") or "").strip()if not job_id:@@ -116,6 +112,20 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])merge_batches = bool(run.get("merge_batches"))ensure_dir(_ARTIFACT_ROOT)run_config_raw = run.get("run_config") or {}+ execution_type = str(run.get("execution_type") or "full").strip().lower()+ planned_artifact_path = (+ str(_ARTIFACT_ROOT / dataset_name) if execution_type != "preview" else None+ )+ event_queue.put(+ {+ "type": EVENT_JOB_STARTED,+ "ts": time.time(),+ "artifact_path": planned_artifact_path,+ "execution_type": execution_type,+ }+ )++ from data_designer.config.run_config import RunConfigbuilder = build_config_builder(recipe)designer = create_data_designer(recipe, artifact_path = str(_ARTIFACT_ROOT))@@ -138,7 +148,6 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])if run_config_raw:designer.set_run_config(RunConfig.model_validate(run_config_raw))- execution_type = str(run.get("execution_type") or "full").strip().lower()if execution_type == "preview":results = designer.preview(builder, num_records = rows)analysis = (diff --git a/studio/backend/tests/test_data_recipe_live_dataset.py b/studio/backend/tests/test_data_recipe_live_dataset.pynew file mode 100644index 00000000000..2602c94925e--- /dev/null+++ b/studio/backend/tests/test_data_recipe_live_dataset.py@@ -0,0 +1,101 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Rows of an active full run are readable once a batch lands in parquet-files."""++from __future__ import annotations++import sys+import threading+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++from core.data_recipe.jobs.constants import EVENT_JOB_STARTED # noqa: E402+from core.data_recipe.jobs.manager import JobManager # noqa: E402+from core.data_recipe.jobs.types import Job # noqa: E402++pd = pytest.importorskip("pandas")+pytest.importorskip("pyarrow")+pytest.importorskip("duckdb")+++def _manager(status: str) -> JobManager:+ m = JobManager.__new__(JobManager)+ m._lock = threading.Lock()+ m._job = Job(job_id = "job-live")+ m._job.status = status+ m._emit = lambda event: None+ return m+++def test_started_event_publishes_planned_artifact_path(tmp_path):+ m = _manager("pending")+ planned = str(tmp_path / "recipe_x")+ m._handle_event(+ m._job,+ {"type": EVENT_JOB_STARTED, "artifact_path": planned, "execution_type": "full"},+ )+ assert m._job.status == "active"+ assert m._job.artifact_path == planned+ assert m._job.execution_type == "full"+++def test_active_run_reads_landed_batches(tmp_path):+ m = _manager("active")+ base = tmp_path / "recipe_x"+ m._job.artifact_path = str(base)+ assert m.get_dataset("job-live", limit = 10) is None++ parquet_dir = base / "parquet-files"+ parquet_dir.mkdir(parents = True)+ pd.DataFrame({"a": [1, 2, 3]}).to_parquet(parquet_dir / "batch_00000.parquet", index = False)+ page = m.get_dataset("job-live", limit = 10)+ assert page is not None and "error" not in page+ assert page["total"] == 3+ assert [row["a"] for row in page["dataset"]] == [1, 2, 3]++[email protected]("status", ["completed", "error", "cancelled"])+def test_finished_run_without_parquet_is_an_error(tmp_path, status):+ m = _manager(status)+ m._job.artifact_path = str(tmp_path / "recipe_x")+ assert "error" in m.get_dataset("job-live", limit = 10)+++def test_active_run_never_uses_the_fallback_reader(tmp_path, monkeypatch):+ m = _manager("active")+ parquet_dir = tmp_path / "recipe_x" / "parquet-files"+ parquet_dir.mkdir(parents = True)+ m._job.artifact_path = str(parquet_dir.parent)+ monkeypatch.setattr(+ JobManager, "_load_dataset_page_with_duckdb", staticmethod(lambda **_: None)+ )++ def _fallback(**_):+ raise AssertionError("fallback reader used during an active run")++ monkeypatch.setattr(+ JobManager, "_load_dataset_page_with_data_designer", staticmethod(_fallback)+ )+ assert m.get_dataset("job-live", limit = 10) is None++[email protected]("offset,limit", [(0, 5), (3, 6), (9, 4), (12, 20), (20, 5)])+def test_duckdb_page_matches_shards_read_in_order(tmp_path, offset, limit):+ sizes = [4, 5, 3, 7]+ start = 0+ for i, n in enumerate(sizes):+ pd.DataFrame({"a": list(range(start, start + n))}).to_parquet(+ tmp_path / f"batch_{i:05d}.parquet", index = False+ )+ start += n+ page = JobManager._load_dataset_page_with_duckdb(+ parquet_dir = tmp_path, limit = limit, offset = offset+ )+ assert page["total"] == sum(sizes)+ assert [row["a"] for row in page["dataset"]] == list(range(sum(sizes)))[offset : offset + limit]diff --git a/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx b/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsxindex 1616f6f77de..3db7800021c 100644--- a/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx+++ b/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx@@ -177,7 +177,14 @@ export function ExecutionDataTab({return (<div className="mt-3"><div className="mb-2 flex flex-wrap items-center justify-between gap-2">- <p className="text-sm font-semibold">Dataset sample</p>+ <div className="space-y-0.5">+ <p className="text-sm font-semibold">Dataset sample</p>+ {isExecutionInProgress(execution.status) && execution.dataset.length > 0 && (+ <p className="text-xs text-muted-foreground">+ Live preview: new rows appear as each batch finishes.+ </p>+ )}+ </div><div className="flex items-center gap-2 text-xs text-muted-foreground">{datasetColumnNames.length > 0 && (<DropdownMenu>diff --git a/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx b/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsxindex 7a193be583b..61087091430 100644--- a/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx+++ b/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx@@ -243,7 +243,9 @@ export function ExecutionsView({: Math.max(1, Math.ceil(datasetTotal / datasetPageSize));const canPageDataset =selectedExecution?.kind === "preview" ||- (selectedExecution?.kind === "full" && Boolean(selectedExecution.jobId));+ (selectedExecution?.kind === "full" &&+ Boolean(selectedExecution.jobId) &&+ !isExecutionInProgress(selectedExecution.status));const datasetRowsForTable = useMemo(() => {if (!selectedExecution) {return [];diff --git a/studio/frontend/src/features/recipe-studio/executions/tracker.ts b/studio/frontend/src/features/recipe-studio/executions/tracker.tsindex 9666e2681fe..0969754988d 100644--- a/studio/frontend/src/features/recipe-studio/executions/tracker.ts+++ b/studio/frontend/src/features/recipe-studio/executions/tracker.ts@@ -104,6 +104,7 @@ export async function trackRecipeExecution({let lastStatus: RecipeExecutionStatus = initialExecution.status;let completedEventPayload: Record<string, unknown> | null = null;let latestExecution: RecipeExecutionRecord = initialExecution;+ let lastDatasetPollAt = 0;const eventsAbortController = new AbortController();void streamRecipeJobEvents({@@ -220,6 +221,35 @@ export async function trackRecipeExecution({onUpsert(latestExecution);done = isTerminalStatus(mappedStatus);+ if (!done && Date.now() - lastDatasetPollAt >= 2500) {+ lastDatasetPollAt = Date.now();+ try {+ const datasetPage = Math.max(1, latestExecution.datasetPage ?? 1);+ const datasetPageSize =+ latestExecution.datasetPageSize ?? DATASET_PAGE_SIZE;+ const datasetResponse = await getRecipeJobDataset(jobId, {+ limit: datasetPageSize,+ offset: (datasetPage - 1) * datasetPageSize,+ });+ const dataset = normalizeDatasetRows(datasetResponse.dataset);+ const datasetTotal =+ typeof datasetResponse.total === "number"+ ? datasetResponse.total+ : dataset.length;+ if (dataset.length > 0 || datasetTotal > 0) {+ latestExecution = {+ ...latestExecution,+ dataset,+ datasetTotal,+ datasetPage,+ datasetPageSize,+ };+ onUpsert(latestExecution);+ }+ } catch {+ // 404 until the first batch lands.+ }+ }if (!done) {await delay(1200);}