Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: live dataset preview during active runs (#10737)

Sourav Rajvi25fabecd6fb3
diff --git a/studio/backend/core/data_recipe/jobs/manager.py b/studio/backend/core/data_recipe/jobs/manager.py
index ab2850a8109..93cb03eda60 100644
--- a/studio/backend/core/data_recipe/jobs/manager.py
+++ b/studio/backend/core/data_recipe/jobs/manager.py
@@ -359,7 +359,15 @@ class JobManager:
base_dataset_path = Path(artifact_path)
parquet_dir = base_dataset_path / "parquet-files"
if not parquet_dir.exists():
- return {"error": f"dataset path missing: {parquet_dir}"}
+ if job_status in {"completed", "error", "cancelled"}:
+ return {"error": f"dataset path missing: {parquet_dir}"}
+ return None
+ if job_status not in {"completed", "error", "cancelled"}:
+ # DuckDB opens with FILE_SHARE_DELETE; the pyarrow fallback would block the
+ # worker's merge rmtree on Windows.
+ return self._load_dataset_page_with_duckdb(
+ parquet_dir = parquet_dir, limit = limit, offset = offset
+ )
return self._load_dataset_page(parquet_dir = parquet_dir, limit = limit, offset = offset)
except Exception as exc:
@@ -393,29 +401,36 @@ class JobManager:
try:
conn = duckdb.connect(":memory:")
try:
- total_row = conn.execute(
- "SELECT COUNT(*) FROM read_parquet(?)",
+ # Row counts from the footers, then scan only the shards the page covers: a full
+ # sort per request grows with the run and the live preview polls page 1.
+ shards = conn.execute(
+ "SELECT file_name, num_rows FROM parquet_file_metadata(?) ORDER BY file_name",
[parquet_glob],
- ).fetchone()
- total = int(total_row[0] if total_row else 0)
- dataframe = conn.execute(
- (
- "SELECT *, row_number() OVER (PARTITION BY filename) AS __row_num__ "
- "FROM read_parquet(?, filename=true) "
- "ORDER BY filename, __row_num__ "
- "LIMIT ? OFFSET ?"
- ),
- [parquet_glob, int(limit), int(offset)],
- ).fetchdf()
+ ).fetchall()
+ total = sum(int(num_rows) for _, num_rows in shards)
+ frames = []
+ skip, remaining = int(offset), int(limit)
+ for file_name, num_rows in shards:
+ if remaining <= 0:
+ break
+ if skip >= int(num_rows):
+ skip -= int(num_rows)
+ continue
+ frame = conn.execute(
+ "SELECT * FROM read_parquet(?) LIMIT ? OFFSET ?",
+ [file_name, remaining, skip],
+ ).fetchdf()
+ frames.append(frame)
+ remaining -= len(frame.index)
+ skip = 0
finally:
conn.close()
except (RuntimeError, ValueError, duckdb.Error):
return None
- for helper_col in ("filename", "__row_num__"):
- if helper_col in dataframe.columns:
- dataframe = dataframe.drop(columns = [helper_col])
+ import pandas as pd
+ dataframe = pd.concat(frames, ignore_index = True) if frames else pd.DataFrame()
rows = dataframe.to_dict(orient = "records")
return {"dataset": to_preview_jsonable_row(rows), "total": total}
@@ -587,6 +602,8 @@ class JobManager:
return
if et == EVENT_JOB_STARTED:
self._job.status = "active"
+ self._job.artifact_path = event.get("artifact_path") or self._job.artifact_path
+ self._job.execution_type = event.get("execution_type") or self._job.execution_type
if et == EVENT_JOB_COMPLETED:
self._job.status = "completed"
self._job.finished_at = time.time()
diff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.py
index 9b8b04d9d7f..b4c5c4d5bdb 100644
--- a/studio/backend/core/data_recipe/jobs/worker.py
+++ b/studio/backend/core/data_recipe/jobs/worker.py
@@ -97,11 +97,7 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])
env = os.getenv("ENVIRONMENT_TYPE", "production"),
)
- event_queue.put({"type": EVENT_JOB_STARTED, "ts": time.time()})
-
try:
- from data_designer.config.run_config import RunConfig
-
rows = int(run.get("rows") or 1000)
job_id = str(run.get("_job_id") or "").strip()
if not job_id:
@@ -116,6 +112,20 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])
merge_batches = bool(run.get("merge_batches"))
ensure_dir(_ARTIFACT_ROOT)
run_config_raw = run.get("run_config") or {}
+ execution_type = str(run.get("execution_type") or "full").strip().lower()
+ planned_artifact_path = (
+ str(_ARTIFACT_ROOT / dataset_name) if execution_type != "preview" else None
+ )
+ event_queue.put(
+ {
+ "type": EVENT_JOB_STARTED,
+ "ts": time.time(),
+ "artifact_path": planned_artifact_path,
+ "execution_type": execution_type,
+ }
+ )
+
+ from data_designer.config.run_config import RunConfig
builder = build_config_builder(recipe)
designer = create_data_designer(recipe, artifact_path = str(_ARTIFACT_ROOT))
@@ -138,7 +148,6 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])
if run_config_raw:
designer.set_run_config(RunConfig.model_validate(run_config_raw))
- execution_type = str(run.get("execution_type") or "full").strip().lower()
if execution_type == "preview":
results = designer.preview(builder, num_records = rows)
analysis = (
diff --git a/studio/backend/tests/test_data_recipe_live_dataset.py b/studio/backend/tests/test_data_recipe_live_dataset.py
new file mode 100644
index 00000000000..2602c94925e
--- /dev/null
+++ b/studio/backend/tests/test_data_recipe_live_dataset.py
@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Rows of an active full run are readable once a batch lands in parquet-files."""
+
+from __future__ import annotations
+
+import sys
+import threading
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from core.data_recipe.jobs.constants import EVENT_JOB_STARTED # noqa: E402
+from core.data_recipe.jobs.manager import JobManager # noqa: E402
+from core.data_recipe.jobs.types import Job # noqa: E402
+
+pd = pytest.importorskip("pandas")
+pytest.importorskip("pyarrow")
+pytest.importorskip("duckdb")
+
+
+def _manager(status: str) -> JobManager:
+ m = JobManager.__new__(JobManager)
+ m._lock = threading.Lock()
+ m._job = Job(job_id = "job-live")
+ m._job.status = status
+ m._emit = lambda event: None
+ return m
+
+
+def test_started_event_publishes_planned_artifact_path(tmp_path):
+ m = _manager("pending")
+ planned = str(tmp_path / "recipe_x")
+ m._handle_event(
+ m._job,
+ {"type": EVENT_JOB_STARTED, "artifact_path": planned, "execution_type": "full"},
+ )
+ assert m._job.status == "active"
+ assert m._job.artifact_path == planned
+ assert m._job.execution_type == "full"
+
+
+def test_active_run_reads_landed_batches(tmp_path):
+ m = _manager("active")
+ base = tmp_path / "recipe_x"
+ m._job.artifact_path = str(base)
+ assert m.get_dataset("job-live", limit = 10) is None
+
+ parquet_dir = base / "parquet-files"
+ parquet_dir.mkdir(parents = True)
+ pd.DataFrame({"a": [1, 2, 3]}).to_parquet(parquet_dir / "batch_00000.parquet", index = False)
+ page = m.get_dataset("job-live", limit = 10)
+ assert page is not None and "error" not in page
+ assert page["total"] == 3
+ assert [row["a"] for row in page["dataset"]] == [1, 2, 3]
+
+
[email protected]("status", ["completed", "error", "cancelled"])
+def test_finished_run_without_parquet_is_an_error(tmp_path, status):
+ m = _manager(status)
+ m._job.artifact_path = str(tmp_path / "recipe_x")
+ assert "error" in m.get_dataset("job-live", limit = 10)
+
+
+def test_active_run_never_uses_the_fallback_reader(tmp_path, monkeypatch):
+ m = _manager("active")
+ parquet_dir = tmp_path / "recipe_x" / "parquet-files"
+ parquet_dir.mkdir(parents = True)
+ m._job.artifact_path = str(parquet_dir.parent)
+ monkeypatch.setattr(
+ JobManager, "_load_dataset_page_with_duckdb", staticmethod(lambda **_: None)
+ )
+
+ def _fallback(**_):
+ raise AssertionError("fallback reader used during an active run")
+
+ monkeypatch.setattr(
+ JobManager, "_load_dataset_page_with_data_designer", staticmethod(_fallback)
+ )
+ assert m.get_dataset("job-live", limit = 10) is None
+
+
[email protected]("offset,limit", [(0, 5), (3, 6), (9, 4), (12, 20), (20, 5)])
+def test_duckdb_page_matches_shards_read_in_order(tmp_path, offset, limit):
+ sizes = [4, 5, 3, 7]
+ start = 0
+ for i, n in enumerate(sizes):
+ pd.DataFrame({"a": list(range(start, start + n))}).to_parquet(
+ tmp_path / f"batch_{i:05d}.parquet", index = False
+ )
+ start += n
+ page = JobManager._load_dataset_page_with_duckdb(
+ parquet_dir = tmp_path, limit = limit, offset = offset
+ )
+ assert page["total"] == sum(sizes)
+ assert [row["a"] for row in page["dataset"]] == list(range(sum(sizes)))[offset : offset + limit]
diff --git a/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx b/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx
index 1616f6f77de..3db7800021c 100644
--- a/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx
+++ b/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx
@@ -177,7 +177,14 @@ export function ExecutionDataTab({
return (
<div className="mt-3">
<div className="mb-2 flex flex-wrap items-center justify-between gap-2">
- <p className="text-sm font-semibold">Dataset sample</p>
+ <div className="space-y-0.5">
+ <p className="text-sm font-semibold">Dataset sample</p>
+ {isExecutionInProgress(execution.status) && execution.dataset.length > 0 && (
+ <p className="text-xs text-muted-foreground">
+ Live preview: new rows appear as each batch finishes.
+ </p>
+ )}
+ </div>
<div className="flex items-center gap-2 text-xs text-muted-foreground">
{datasetColumnNames.length > 0 && (
<DropdownMenu>
diff --git a/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx b/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx
index 7a193be583b..61087091430 100644
--- a/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx
+++ b/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx
@@ -243,7 +243,9 @@ export function ExecutionsView({
: Math.max(1, Math.ceil(datasetTotal / datasetPageSize));
const canPageDataset =
selectedExecution?.kind === "preview" ||
- (selectedExecution?.kind === "full" && Boolean(selectedExecution.jobId));
+ (selectedExecution?.kind === "full" &&
+ Boolean(selectedExecution.jobId) &&
+ !isExecutionInProgress(selectedExecution.status));
const datasetRowsForTable = useMemo(() => {
if (!selectedExecution) {
return [];
diff --git a/studio/frontend/src/features/recipe-studio/executions/tracker.ts b/studio/frontend/src/features/recipe-studio/executions/tracker.ts
index 9666e2681fe..0969754988d 100644
--- a/studio/frontend/src/features/recipe-studio/executions/tracker.ts
+++ b/studio/frontend/src/features/recipe-studio/executions/tracker.ts
@@ -104,6 +104,7 @@ export async function trackRecipeExecution({
let lastStatus: RecipeExecutionStatus = initialExecution.status;
let completedEventPayload: Record<string, unknown> | null = null;
let latestExecution: RecipeExecutionRecord = initialExecution;
+ let lastDatasetPollAt = 0;
const eventsAbortController = new AbortController();
void streamRecipeJobEvents({
@@ -220,6 +221,35 @@ export async function trackRecipeExecution({
onUpsert(latestExecution);
done = isTerminalStatus(mappedStatus);
+ if (!done && Date.now() - lastDatasetPollAt >= 2500) {
+ lastDatasetPollAt = Date.now();
+ try {
+ const datasetPage = Math.max(1, latestExecution.datasetPage ?? 1);
+ const datasetPageSize =
+ latestExecution.datasetPageSize ?? DATASET_PAGE_SIZE;
+ const datasetResponse = await getRecipeJobDataset(jobId, {
+ limit: datasetPageSize,
+ offset: (datasetPage - 1) * datasetPageSize,
+ });
+ const dataset = normalizeDatasetRows(datasetResponse.dataset);
+ const datasetTotal =
+ typeof datasetResponse.total === "number"
+ ? datasetResponse.total
+ : dataset.length;
+ if (dataset.length > 0 || datasetTotal > 0) {
+ latestExecution = {
+ ...latestExecution,
+ dataset,
+ datasetTotal,
+ datasetPage,
+ datasetPageSize,
+ };
+ onUpsert(latestExecution);
+ }
+ } catch {
+ // 404 until the first batch lands.
+ }
+ }
if (!done) {
await delay(1200);
}