Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: extract ebooks, Office files and email in Data Recipes document seeds (#14499)
Daniel Hanfee26fe12a07
diff --git a/studio/backend/core/rag/parsers.py b/studio/backend/core/rag/parsers.pyindex 1b4c1cb60eb..b6fe7bd0422 100644--- a/studio/backend/core/rag/parsers.py+++ b/studio/backend/core/rag/parsers.py@@ -531,6 +531,22 @@ def parse_pdf_bytes(data: bytes, *, max_pages: int | None = None) -> tuple[list[return pages, total_pages+def mupdf_document_text(path: str) -> str:+ """Paragraph text of a reflowable e-book only MuPDF reads (.mobi, .fb2)."""+ import pymupdf++ with pymupdf.open(path) as doc:+ # A tall page keeps paragraphs from splitting at page breaks.+ doc.layout(width = 1000, height = 100000, fontsize = 11)+ paragraphs = [+ " ".join(block[4].split())+ for page in doc+ for block in page.get_text("blocks")+ if block[6] == 0+ ]+ return "\n\n".join(p for p in paragraphs if p)++def _merge_rects(boxes: list) -> list:"""Union overlapping rectangles (largest-first) into figure regions."""import pymupdfdiff --git a/studio/backend/models/data_recipe.py b/studio/backend/models/data_recipe.pyindex b9b89ca82bb..3e5aa129bc7 100644--- a/studio/backend/models/data_recipe.py+++ b/studio/backend/models/data_recipe.py@@ -115,6 +115,7 @@ class SeedInspectResponse(BaseModel):split: str | None = Nonesubset: str | None = Noneresolved_paths: list[str] | None = None+ total_rows: int | None = Noneclass UnstructuredFileUploadResponse(BaseModel):diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.pyindex 2256c11ace1..5362f541c6b 100644--- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py+++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py@@ -69,6 +69,22 @@ def build_multi_file_preview_rows(chunk_size: int | None,chunk_overlap: int | None,) -> list[dict[str, str]]:+ return build_multi_file_preview(+ file_entries = file_entries,+ preview_size = preview_size,+ chunk_size = chunk_size,+ chunk_overlap = chunk_overlap,+ )[0]+++def build_multi_file_preview(+ *,+ file_entries: list[tuple[Path, str]],+ preview_size: int,+ chunk_size: int | None,+ chunk_overlap: int | None,+) -> tuple[list[dict[str, str]], int]:+ """Preview rows plus the total chunk count, from one materialization."""cs = _to_int(chunk_size, DEFAULT_CHUNK_SIZE)co = _to_int(chunk_overlap, DEFAULT_CHUNK_OVERLAP)_, rows = materialize_multi_file_unstructured_seed(@@ -76,7 +92,7 @@ def build_multi_file_preview_rows(chunk_size = cs,chunk_overlap = co,)- return _round_robin_preview(rows, preview_size)+ return _round_robin_preview(rows, preview_size), len(rows)def _round_robin_preview(rows: list[dict[str, str]], preview_size: int) -> list[dict[str, str]]:diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.pyindex 12930eba139..79a700f0dc6 100644--- a/studio/backend/routes/data_recipe/seed.py+++ b/studio/backend/routes/data_recipe/seed.py@@ -17,7 +17,7 @@ from pathlib import Path, PurePosixPathfrom typing import Anyfrom uuid import uuid4-from anyio import CapacityLimiter, to_process+from anyio import CapacityLimiter, to_process, to_threadfrom fastapi import APIRouter, Depends, HTTPException, UploadFile, File as FastAPIFile, Formfrom auth.authentication import allow_ambient_hf_token@@ -78,7 +78,17 @@ DATA_EXTS = (".parquet", ".jsonl", ".json", ".csv")DEFAULT_SPLIT = "train"DEFAULT_CONFIG = "default"LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl", ".parquet"}-UNSTRUCTURED_ALLOWED_EXTS = {".pdf", ".docx", ".txt", ".md"}+# Each type needs a core.rag parser or MuPDF; a frontend parity test reads this literal.+UNSTRUCTURED_ALLOWED_EXTS = {+ ".pdf", ".docx", ".txt", ".md", ".markdown", ".html", ".htm", ".xhtml", ".xht",+ ".doc", ".docm", ".dotx", ".dotm", ".odt", ".ott", ".rtf",+ ".epub", ".mobi", ".fb2",+ ".xlsx", ".xlsm", ".xltx", ".xltm", ".xls", ".xlt", ".ods", ".ots",+ ".pptx", ".pptm", ".potx", ".potm", ".ppsx", ".ppsm", ".ppt", ".pps", ".odp", ".otp",+ ".eml", ".msg", ".mht", ".mhtml",+} # fmt: skip+# MuPDF is not thread-safe, so these run in the PDF worker pool.+_MUPDF_EBOOK_EXTS = {".mobi", ".fb2"}SEED_UPLOAD_DIR = LazyPath(seed_uploads_root)UNSTRUCTURED_UPLOAD_ROOT = LazyPath(unstructured_uploads_root)_SAFE_ID_RE = re.compile(r"^[a-zA-Z0-9_-]+$")@@ -1082,7 +1092,7 @@ def _read_preview_rows_from_multi_files(preview_size: int,chunk_size: int | None,chunk_overlap: int | None,-) -> list[dict[str, str]]:+) -> tuple[list[dict[str, str]], int]:chunking = _chunking()if chunking is None:raise HTTPException(@@ -1090,6 +1100,17 @@ def _read_preview_rows_from_multi_files("Unstructured seed support not available (missing data_designer_unstructured_seed)",)+ return chunking.build_multi_file_preview(+ file_entries = _multi_file_entries(block_id, file_ids, file_names),+ preview_size = preview_size,+ chunk_size = chunk_size,+ chunk_overlap = chunk_overlap,+ )+++def _multi_file_entries(+ block_id: str, file_ids: list[str], file_names: list[str]+) -> list[tuple[Path, str]]:_validate_safe_id(block_id, "block_id")block_dir = UNSTRUCTURED_UPLOAD_ROOT / block_idfile_entries: list[tuple[Path, str]] = []@@ -1098,13 +1119,7 @@ def _read_preview_rows_from_multi_files(if not extracted.exists():raise HTTPException(404, f"Extracted text not found for file: {fname} (id: {fid})")file_entries.append((extracted, fname))-- return chunking.build_multi_file_preview_rows(- file_entries = file_entries,- preview_size = preview_size,- chunk_size = chunk_size,- chunk_overlap = chunk_overlap,- )+ return file_entriesdef _require_own_seed_source(dataset_name: str) -> None:@@ -1254,7 +1269,10 @@ def _extract_text_from_file(file_path: Path, ext: str) -> str:elif ext == ".pdf":from core.rag import config, pdf_ocrraw = pdf_ocr.extract_text(str(file_path), config.OCR_SCANNED, config.OCR_MAX_PAGES)- elif ext == ".docx":+ elif ext in _MUPDF_EBOOK_EXTS:+ from core.rag import parsers+ raw = parsers.mupdf_document_text(str(file_path))+ elif ext in UNSTRUCTURED_ALLOWED_EXTS:from core.rag import parsersraw = "\n\n".join(page.text for page in parsers.parse(str(file_path)))else:@@ -1267,11 +1285,26 @@ def _extract_text_from_file(file_path: Path, ext: str) -> str:_pdf_extraction_limiter = CapacityLimiter(2)+_document_extraction_limiter = CapacityLimiter(2)async def _extract_text_from_file_async(file_path: Path, ext: str) -> str:+ if ext in _MUPDF_EBOOK_EXTS:+ from core.rag import parsers++ raw = await to_process.run_sync(+ parsers.mupdf_document_text,+ str(file_path),+ cancellable = True,+ limiter = _pdf_extraction_limiter,+ )+ chunking = _chunking()+ return chunking.normalize_unstructured_text(raw) if chunking is not None else rawif ext != ".pdf":- return _extract_text_from_file(file_path, ext)+ # A large spreadsheet must not stall the event loop, nor several fill the shared thread pool.+ return await to_thread.run_sync(+ _extract_text_from_file, file_path, ext, limiter = _document_extraction_limiter+ )from core.rag import config, pdf_ocr# MuPDF is not thread-safe; each worker owns its document and OCR state.@@ -1430,7 +1463,11 @@ async def upload_unstructured_file(raw_path.unlink(missing_ok = True)extracted_path.unlink(missing_ok = True)missing = getattr(e, "name", None)- expected_missing = {".pdf": "pymupdf4llm", ".docx": "docx"}.get(ext)+ expected_missing = {+ ".pdf": "pymupdf4llm",+ **dict.fromkeys((".docx", ".docm", ".dotx", ".dotm"), "docx"),+ **dict.fromkeys(_MUPDF_EBOOK_EXTS, "pymupdf"),+ }.get(ext)if isinstance(e, ModuleNotFoundError) and missing == expected_missing:# Name what to install: python-docx imports as docx, and PyPI's "docx" is another package.package = {"docx": "python-docx"}.get(missing, missing)@@ -1576,7 +1613,7 @@ def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectRespons_validate_safe_id(payload.block_id, "block_id")for fid in payload.file_ids:_validate_safe_id(fid, "file_id")- preview_rows = _read_preview_rows_from_multi_files(+ preview_rows, total_rows = _read_preview_rows_from_multi_files(block_id = payload.block_id,file_ids = payload.file_ids,file_names = payload.file_names,@@ -1595,6 +1632,7 @@ def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectResponsresolved_paths = resolved_paths,columns = columns,preview_rows = _serialize_preview_rows(preview_rows),+ total_rows = total_rows,)seed_source_type = _normalize_optional_text(payload.seed_source_type) or "local"diff --git a/studio/backend/tests/test_data_recipe_books_template.py b/studio/backend/tests/test_data_recipe_books_template.pynew file mode 100644index 00000000000..d3f78b38517--- /dev/null+++ b/studio/backend/tests/test_data_recipe_books_template.py@@ -0,0 +1,35 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import json+from pathlib import Path++import pytest++_TEMPLATE = (+ Path(__file__).resolve().parents[2]+ / "frontend/src/features/data-recipes/learning-recipes/books-and-documents.json"+)+++def test_books_template_turns_extracted_text_into_text_rows_without_a_model(tmp_path):+ pytest.importorskip("data_designer")+ pytest.importorskip("data_designer_unstructured_seed")+ from core.data_recipe.service import build_config_builder, create_data_designer++ paragraph = "It was a bright cold day in April, and the clocks were striking thirteen. "+ extracted = tmp_path / "book.extracted.txt"+ extracted.write_text("\n\n".join(paragraph * 20 for _ in range(10)), encoding = "utf-8")+ recipe = json.loads(_TEMPLATE.read_text(encoding = "utf-8"))["recipe"]+ assert recipe["model_providers"] == [] and recipe["model_configs"] == []+ source = recipe["seed_config"]["source"]+ source.pop("path")+ source["paths"] = [str(extracted)]++ designer = create_data_designer(recipe, artifact_path = str(tmp_path / "artifacts"))+ rows = designer.preview(build_config_builder(recipe), num_records = 2).dataset++ assert list(rows.columns) == ["source_file", "text"]+ assert len(rows) == 2+ assert rows["text"][0].startswith("It was a bright cold day")+ assert all(0 < len(text) <= source["chunk_size"] for text in rows["text"])diff --git a/studio/backend/tests/test_data_recipe_seed.py b/studio/backend/tests/test_data_recipe_seed.pyindex 185d6ab5205..63f60888213 100644--- a/studio/backend/tests/test_data_recipe_seed.py+++ b/studio/backend/tests/test_data_recipe_seed.py@@ -1781,3 +1781,119 @@ def test_validate_answers_when_the_error_collector_cannot_read_the_seed(monkeypaassert response.valid is Falseassert "repository not found" in (response.raw_detail or "")+++def test_every_recipe_document_type_has_an_extractor(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ from core.rag import config++ readable = (+ config.SUPPORTED_UPLOAD_EXTS | config.DOCUMENT_UPLOAD_EXTS | seed_route._MUPDF_EBOOK_EXTS+ )+ assert seed_route.UNSTRUCTURED_ALLOWED_EXTS <= readable+ assert {".epub", ".mobi", ".doc", ".xlsx", ".pptx", ".odt", ".rtf", ".eml"} <= (+ seed_route.UNSTRUCTURED_ALLOWED_EXTS+ )+++def test_recipe_upload_extracts_office_ebook_and_email_files(monkeypatch, tmp_path):+ from .test_rag_office_formats import BUILDERS+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ for ext, build in sorted(BUILDERS.items()):+ assert ext in seed_route.UNSTRUCTURED_ALLOWED_EXTS, ext+ source = build(tmp_path / f"source{ext}")+ result = _run_upload(seed_route, f"book{ext}", source.read_bytes(), block_id = ext[1:])+ assert result.status == "ok", (ext, result.error)+ extracted = (+ seed_route.UNSTRUCTURED_UPLOAD_ROOT / ext[1:] / f"{result.file_id}.extracted.txt"+ )+ assert "zebramarker" in extracted.read_text(encoding = "utf-8").lower(), ext+++_FB2 = (+ '<?xml version="1.0" encoding="utf-8"?>'+ '<FictionBook xmlns="http://www.gribuser.ru/xml/fictionbook/2.0">'+ "<description><title-info><book-title>Tiny</book-title></title-info></description>"+ "<body><section><title><p>Chapter One</p></title>"+ "<p>It was a bright cold day in April, and the clocks were striking thirteen.</p>"+ "<p>Zebramarker closes the chapter.</p></section></body></FictionBook>"+).encode()+++def test_recipe_upload_reads_mupdf_ebooks_in_the_worker_pool(monkeypatch, tmp_path):+ pytest.importorskip("pymupdf")+ seed_route = _load_seed_route(monkeypatch, tmp_path, inline_extraction = False)++ result = _run_upload(seed_route, "novel.fb2", _FB2)++ assert result.status == "ok", result.error+ extracted = seed_route.UNSTRUCTURED_UPLOAD_ROOT / "block" / f"{result.file_id}.extracted.txt"+ assert extracted.read_text(encoding = "utf-8") == (+ "Chapter One\n\n"+ "It was a bright cold day in April, and the clocks were striking thirteen.\n\n"+ "Zebramarker closes the chapter."+ )+++def test_recipe_upload_names_pymupdf_when_an_ebook_reader_is_missing(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ monkeypatch.setattr(+ seed_route,+ "_extract_text_from_file",+ _raise(ModuleNotFoundError("No module named 'pymupdf'", name = "pymupdf")),+ )++ result = _run_upload(seed_route, "novel.mobi", b"BOOKMOBI")++ assert result.status == "error"+ assert result.error == "Cannot read .mobi files: the 'pymupdf' package is not installed."+ assert _block_files(seed_route) == []+++def test_document_seed_preview_reports_every_chunk_not_just_the_preview(monkeypatch, tmp_path):+ chunking = pytest.importorskip("data_designer_unstructured_seed.chunking")+ from models.data_recipe import SeedInspectUploadRequest++ seed_route = _load_seed_route(monkeypatch, tmp_path)+ monkeypatch.setattr(chunking, "_CACHE_DIR", tmp_path / "chunk-cache")+ sentence = "A sentence long enough to fill most of a small chunk on its own. "+ ids, names = [], []+ for name in ("one.txt", "two.txt"):+ result = _run_upload(seed_route, name, ("\n\n".join([sentence] * 30)).encode())+ ids.append(result.file_id)+ names.append(name)++ response = seed_route.inspect_seed_upload(+ SeedInspectUploadRequest(+ block_id = "block",+ file_ids = ids,+ file_names = names,+ preview_size = 3,+ seed_source_type = "unstructured",+ unstructured_chunk_size = 100,+ unstructured_chunk_overlap = 0,+ )+ )++ assert len(response.preview_rows) == 3+ assert response.total_rows == 60+++def test_document_extraction_threads_are_bounded(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path, inline_extraction = False)+ seen = []++ async def run_sync(+ fn,+ *args,+ limiter = None,+ **kwargs,+ ):+ seen.append(limiter)+ return fn(*args)++ monkeypatch.setattr(seed_route.to_thread, "run_sync", run_sync)++ assert _run_upload(seed_route, "notes.txt", b"hello").status == "ok"+ assert seen == [seed_route._document_extraction_limiter]+ assert seed_route._document_extraction_limiter.total_tokens == 2diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/books-and-documents.json b/studio/frontend/src/features/data-recipes/learning-recipes/books-and-documents.jsonnew file mode 100644index 00000000000..612efa6f89d--- /dev/null+++ b/studio/frontend/src/features/data-recipes/learning-recipes/books-and-documents.json@@ -0,0 +1,88 @@+{+ "recipe": {+ "model_providers": [],+ "mcp_providers": [],+ "model_configs": [],+ "seed_config": {+ "source": {+ "seed_type": "unstructured",+ "path": "",+ "chunk_size": 6000,+ "chunk_overlap": 600+ },+ "sampling_strategy": "ordered",+ "selection_strategy": null+ },+ "tool_configs": [],+ "columns": [+ {+ "column_type": "expression",+ "name": "text",+ "drop": false,+ "expr": "{{ chunk_text }}",+ "dtype": "str"+ }+ ],+ "processors": [+ {+ "processor_type": "drop_columns",+ "name": "drop_seed_columns",+ "column_names": [+ "chunk_text"+ ]+ }+ ]+ },+ "run": {+ "rows": 10,+ "preview": true,+ "output_formats": [+ "jsonl"+ ]+ },+ "ui": {+ "nodes": [+ {+ "id": "note_1",+ "x": 0,+ "y": 0,+ "width": 420,+ "node_type": "markdown_note",+ "name": "note_1",+ "markdown": "### Books & Documents to Training Text\nTurn your own ebooks and documents into a plain `text` dataset for continued pretraining or style tuning. No model is needed: every row is a chunk of your files.\n\n**1. Upload files in the seed block.** EPUB, MOBI, FB2, PDF, Word (.docx, .doc), Excel (.xlsx, .xls), PowerPoint (.pptx, .ppt), OpenDocument, RTF, HTML, email (.eml, .msg), TXT and Markdown are extracted by the Studio server you upload to. Images, styles and navigation pages are dropped; paragraphs and chapter order are kept. DRM-protected books cannot be read.\n\n**2. Pick a chunk size.** Chunk size and overlap count characters (about 4 per token). 6000 characters is roughly 1500 tokens; raise it toward your training context length. Chunks break at paragraph and sentence boundaries where possible.\n\n**3. Set rows and Run.** Ordered sampling walks the books from start to end. After you load the files, the seed Preview tab shows how many chunks they make in total; set rows to that number to export each chunk once (more rows start again from the first chunk).\n\nOutput columns: `text` (the chunk) and `source_file` (the original file name). Train on `text` with the dataset text field, or add an LLM column (see the PDF Document QA recipe) to turn each chunk into question and answer pairs.",+ "note_color": "#E0F2FE",+ "note_opacity": "35"+ },+ {+ "id": "seed",+ "x": 500,+ "y": 120,+ "width": 400+ },+ {+ "id": "text",+ "x": 980,+ "y": 120,+ "width": 400+ }+ ],+ "edges": [+ {+ "from": "seed",+ "to": "text",+ "type": "canvas",+ "source_handle": "data-out",+ "target_handle": "data-in"+ }+ ],+ "layout_direction": "LR",+ "seed_source_type": "unstructured",+ "seed_columns": [],+ "seed_drop_columns": [],+ "seed_preview_rows": [],+ "local_file_name": "",+ "unstructured_file_name": "",+ "unstructured_chunk_size": "6000",+ "unstructured_chunk_overlap": "600"+ }+}diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/index.ts b/studio/frontend/src/features/data-recipes/learning-recipes/index.tsindex 8607d3faebd..6b68a75b9c2 100644--- a/studio/frontend/src/features/data-recipes/learning-recipes/index.ts+++ b/studio/frontend/src/features/data-recipes/learning-recipes/index.ts@@ -23,6 +23,10 @@ const githubSupportBotUrl = new URL("./github-support-bot.json",import.meta.url,).href;+const booksAndDocumentsUrl = new URL(+ "./books-and-documents.json",+ import.meta.url,+).href;function isRecord(value: unknown): value is Record<string, unknown> {return !!value && typeof value === "object" && !Array.isArray(value);@@ -148,4 +152,11 @@ export const LEARNING_RECIPES: LearningRecipeDef[] = ["Crawl real GitHub issues and PRs and turn each thread into a {User, Assistant} training pair.",loadPayload: () => loadPayloadFromUrl(githubSupportBotUrl),},+ {+ id: "books-and-documents",+ title: "Books & Documents to Text",+ description:+ "Extract ebooks, PDFs and Office files into a text dataset for continued pretraining.",+ loadPayload: () => loadPayloadFromUrl(booksAndDocumentsUrl),+ },];diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/pdf-grounded-qa.json b/studio/frontend/src/features/data-recipes/learning-recipes/pdf-grounded-qa.jsonindex f911793dd42..436da609691 100644--- a/studio/frontend/src/features/data-recipes/learning-recipes/pdf-grounded-qa.json+++ b/studio/frontend/src/features/data-recipes/learning-recipes/pdf-grounded-qa.json@@ -101,7 +101,7 @@"width": 400,"node_type": "markdown_note","name": "note_1",- "markdown": "This recipe uses **seed data** from external documents.\nInstead of starting from empty generation, we load real source text first.\n\nIn this flow, the seed source is **Unstructured Documents**:\n\n- Upload: `.pdf`, `.docx`, `.txt`\n- Text is extracted and split on client into chunks\n- Each chunk becomes a row-like seed record (`chunk_text`) that you can reference in prompts with `{{ chunk_text }} `",+ "markdown": "This recipe uses **seed data** from external documents.\nInstead of starting from empty generation, we load real source text first.\n\nIn this flow, the seed source is **Unstructured Documents**:\n\n- Upload: `.pdf`, `.docx`, `.epub`, `.mobi`, `.xlsx`, `.pptx`, `.txt` and more\n- Text is extracted and split on client into chunks\n- Each chunk becomes a row-like seed record (`chunk_text`) that you can reference in prompts with `{{ chunk_text }} `","note_color": "#F3E8FF","note_opacity": "35"},diff --git a/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx b/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsxindex 5845c824c97..8369077575a 100644--- a/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx+++ b/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx@@ -31,6 +31,7 @@ import { OPEN_LEARNING_RECIPES_ON_ARRIVAL_KEY } from "@/lib/navigation-intents";import { toastError } from "@/shared/toast";import {Album02Icon,+ BookOpen01Icon,CodeIcon,CookBookIcon,Database02Icon,@@ -180,6 +181,22 @@ const TEMPLATE_CARDS: TemplateCard[] = [],learningRecipeId: "github-support-bot",},+ {+ title: "Books & Documents to Text",+ description:+ "Extract EPUB, MOBI, PDF, Word, Excel, PowerPoint and more into a text dataset for continued pretraining.",+ icon: BookOpen01Icon,+ difficulty: "Easy",+ learningBadges: ["Unstructured", "Ebooks", "Expression"],+ surfaceClassName:+ "from-rose-500/15 via-pink-500/5 to-transparent dark:from-rose-400/20 dark:via-pink-400/10 dark:to-rose-950/16",+ shineColor: [+ "rgb(244 63 94 / 0.45)",+ "rgb(236 72 153 / 0.4)",+ "rgb(251 113 133 / 0.45)",+ ],+ learningRecipeId: "books-and-documents",+ },];const LEARNING_RECIPE_BY_ID = new Map(diff --git a/studio/frontend/src/features/recipe-studio/api/index.ts b/studio/frontend/src/features/recipe-studio/api/index.tsindex 2ab6afea4b5..63de71836bc 100644--- a/studio/frontend/src/features/recipe-studio/api/index.ts+++ b/studio/frontend/src/features/recipe-studio/api/index.ts@@ -167,6 +167,8 @@ export type SeedInspectResponse = {subset?: string | null;// biome-ignore lint/style/useNamingConvention: api schemaresolved_paths?: string[] | null;+ // biome-ignore lint/style/useNamingConvention: api schema+ total_rows?: number | null;};export type ValidateError = {diff --git a/studio/frontend/src/features/recipe-studio/blocks/definitions.ts b/studio/frontend/src/features/recipe-studio/blocks/definitions.tsindex 9f39724db86..7608feee399 100644--- a/studio/frontend/src/features/recipe-studio/blocks/definitions.ts+++ b/studio/frontend/src/features/recipe-studio/blocks/definitions.ts@@ -175,7 +175,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [kind: "seed",type: "seed_unstructured",title: "Document file",- description: "Upload PDF, DOCX, or TXT and turn it into source rows.",+ description: "Upload documents, ebooks, spreadsheets or slides and turn them into source rows.",icon: DocumentAttachmentIcon,dialogKey: "seed",createConfig: (id, existing) => makeSeedConfig(id, existing, "unstructured"),diff --git a/studio/frontend/src/features/recipe-studio/components/block-sheet.tsx b/studio/frontend/src/features/recipe-studio/components/block-sheet.tsxindex 3cde1b34290..520b964f5a7 100644--- a/studio/frontend/src/features/recipe-studio/components/block-sheet.tsx+++ b/studio/frontend/src/features/recipe-studio/components/block-sheet.tsx@@ -586,8 +586,8 @@ export function BlockSheet({Recommended first step</p><p className="text-xs text-muted-foreground">- Best when you want to turn PDFs, DOCX files, or text- files into source rows.+ Best when you want to turn PDFs, ebooks, Office files,+ or text files into source rows.</p></div><BlockSheetButtondiff --git a/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsxindex f0b9fe5ec6d..f6738dd2dfa 100644--- a/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx+++ b/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx@@ -700,6 +700,8 @@ export function SeedDialog({unstructured_file_ids: okFiles.map((f) => f.id),unstructured_file_names: okFiles.map((f) => f.name),unstructured_file_sizes: okFiles.map((f) => f.size),+ // The total is only valid for the files and chunking it was counted with.+ seed_total_rows: undefined,});});return next;@@ -855,6 +857,7 @@ export function SeedDialog({resolved_paths: response.resolved_paths ?? [],seed_columns: response.columns,seed_preview_rows: response.preview_rows ?? [],+ seed_total_rows: response.total_rows ?? undefined,unstructured_file_ids: fileIds,unstructured_file_names: fileNames,unstructured_file_sizes: unstructuredFiles@@ -1206,6 +1209,7 @@ export function SeedDialog({onChange={(event) =>onUpdate({unstructured_chunk_size: event.target.value,+ seed_total_rows: undefined,})}/>@@ -1227,6 +1231,7 @@ export function SeedDialog({onChange={(event) =>onUpdate({unstructured_chunk_overlap: event.target.value,+ seed_total_rows: undefined,})}/>@@ -1323,6 +1328,9 @@ export function SeedDialog({<div className="space-y-2"><div className="text-xs text-muted-foreground">Loaded columns: {previewColumns.join(", ") || "None"}+ {mode === "unstructured" && config.seed_total_rows+ ? ` - ${config.seed_total_rows} chunks in total, ${previewRows.length} shown`+ : null}</div><div className="max-h-[calc(360px*var(--ui-space-scale,1))] overflow-y-auto overflow-x-hidden scroll-rounded rounded-xl corner-squircle border border-border/60"><Table className="corner-squircle min-w-max">diff --git a/studio/frontend/src/features/recipe-studio/dialogs/seed/unstructured-drop-zone.tsx b/studio/frontend/src/features/recipe-studio/dialogs/seed/unstructured-drop-zone.tsxindex fcaf3d33a0e..9f7b31a2cbf 100644--- a/studio/frontend/src/features/recipe-studio/dialogs/seed/unstructured-drop-zone.tsx+++ b/studio/frontend/src/features/recipe-studio/dialogs/seed/unstructured-drop-zone.tsx@@ -16,7 +16,15 @@ import {UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_LABEL,} from "./upload-limits";-const ACCEPTED_EXTENSIONS = [".txt", ".pdf", ".docx", ".md"];+// Mirrors UNSTRUCTURED_ALLOWED_EXTS in routes/data_recipe/seed.py (parity test).+const ACCEPTED_EXTENSIONS = [+ ".pdf", ".docx", ".txt", ".md", ".markdown", ".html", ".htm", ".xhtml", ".xht",+ ".doc", ".docm", ".dotx", ".dotm", ".odt", ".ott", ".rtf",+ ".epub", ".mobi", ".fb2",+ ".xlsx", ".xlsm", ".xltx", ".xltm", ".xls", ".xlt", ".ods", ".ots",+ ".pptx", ".pptm", ".potx", ".potm", ".ppsx", ".ppsm", ".ppt", ".pps", ".odp", ".otp",+ ".eml", ".msg", ".mht", ".mhtml",+];/** One queued upload: browser bytes, or a desktop drop the backend redeems. */type UploadCandidate = {@@ -256,7 +264,7 @@ export function UnstructuredDropZone({Drop files here or click to browse</p><p className="text-muted-foreground/60 mt-1 text-xs">- PDF, DOCX, TXT, MD - up to {UNSTRUCTURED_RECIPE_UPLOAD_MAX_LABEL}{" "}+ PDF, Word, EPUB, MOBI, Excel, PowerPoint, HTML, TXT, MD and more - up to {UNSTRUCTURED_RECIPE_UPLOAD_MAX_LABEL}{" "}each, {UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_LABEL} total</p></div>diff --git a/studio/frontend/src/features/recipe-studio/hooks/use-recipe-persistence.ts b/studio/frontend/src/features/recipe-studio/hooks/use-recipe-persistence.tsindex c1f406fd14a..b48d624f689 100644--- a/studio/frontend/src/features/recipe-studio/hooks/use-recipe-persistence.ts+++ b/studio/frontend/src/features/recipe-studio/hooks/use-recipe-persistence.ts@@ -144,6 +144,7 @@ function sanitizeSeedForShare(payload: unknown): unknown {ui.seed_columns = [];ui.seed_drop_columns = [];ui.seed_preview_rows = [];+ delete ui.seed_total_rows;ui.local_file_name = "";ui.unstructured_upload_uid = "";ui.unstructured_file_ids = [];@@ -166,6 +167,7 @@ function sanitizeSeedForShare(payload: unknown): unknown {ui.seed_columns = [];ui.seed_drop_columns = [];ui.seed_preview_rows = [];+ delete ui.seed_total_rows;ui.local_file_name = "";ui.unstructured_upload_uid = "";ui.unstructured_file_ids = [];diff --git a/studio/frontend/src/features/recipe-studio/types/index.ts b/studio/frontend/src/features/recipe-studio/types/index.tsindex 994f14bf200..9619bd786ad 100644--- a/studio/frontend/src/features/recipe-studio/types/index.ts+++ b/studio/frontend/src/features/recipe-studio/types/index.ts@@ -366,6 +366,8 @@ export type SeedConfig = {resolved_paths?: string[];// ui-onlyseed_preview_rows?: Record<string, unknown>[];+ // ui-only+ seed_total_rows?: number;// ui-only (string for input ergonomics)unstructured_chunk_size?: string;// ui-only (string for input ergonomics)diff --git a/studio/frontend/src/features/recipe-studio/utils/import/importer.ts b/studio/frontend/src/features/recipe-studio/utils/import/importer.tsindex dd564ed8703..2d0f0610564 100644--- a/studio/frontend/src/features/recipe-studio/utils/import/importer.ts+++ b/studio/frontend/src/features/recipe-studio/utils/import/importer.ts@@ -38,6 +38,7 @@ type UiInput = {seed_columns?: unknown;seed_drop_columns?: unknown;seed_preview_rows?: unknown;+ seed_total_rows?: unknown;local_file_name?: unknown;unstructured_upload_uid?: unknown;unstructured_file_ids?: unknown;@@ -421,6 +422,10 @@ export function importRecipePayload(.filter(Boolean): undefined;const payloadSeedDropColumns = parseSeedDropColumns(recipe.processors);+ const uiSeedTotalRows =+ typeof ui?.seed_total_rows === "number" && ui.seed_total_rows > 0+ ? ui.seed_total_rows+ : undefined;const uiSeedPreviewRows = Array.isArray(ui?.seed_preview_rows)? ui.seed_preview_rows.filter((row): row is Record<string, unknown> => isRecord(row))@@ -500,6 +505,7 @@ export function importRecipePayload(? uiSeedDropColumns: payloadSeedDropColumns,seed_preview_rows: uiSeedPreviewRows,+ seed_total_rows: uiSeedTotalRows,local_file_name: uiLocalFileName,unstructuredUploadUid: uiUnstructuredUploadUid,unstructuredFileIds: uiUnstructuredFileIds,diff --git a/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts b/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.tsindex c9717ba1d42..c486fb05e7c 100644--- a/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts+++ b/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts@@ -199,6 +199,7 @@ export function parseSeedConfig(seed_columns?: string[];seed_drop_columns?: string[];seed_preview_rows?: Record<string, unknown>[];+ seed_total_rows?: number;local_file_name?: string;unstructuredUploadUid?: string;unstructuredFileIds?: string[];@@ -240,6 +241,9 @@ export function parseSeedConfig(...(options?.seed_preview_rows? { seed_preview_rows: options.seed_preview_rows }: {}),+ ...(options?.seed_total_rows !== undefined+ ? { seed_total_rows: options.seed_total_rows }+ : {}),...(options?.local_file_name !== undefined? { local_file_name: options.local_file_name }: {}),diff --git a/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts b/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.tsindex 9b2ad5b2b58..fb3e06e7ba9 100644--- a/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts+++ b/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts@@ -430,6 +430,9 @@ export function buildRecipePayload(...(firstSeed && {seed_preview_rows: firstSeed.seed_preview_rows ?? [],}),+ ...(firstSeed?.seed_total_rows !== undefined && {+ seed_total_rows: firstSeed.seed_total_rows,+ }),...(firstSeed &&firstSeed.local_file_name !== undefined && {local_file_name: firstSeed.local_file_name,diff --git a/studio/frontend/src/features/recipe-studio/utils/payload/types.ts b/studio/frontend/src/features/recipe-studio/utils/payload/types.tsindex 763e68c1fbe..a68f0887b8a 100644--- a/studio/frontend/src/features/recipe-studio/utils/payload/types.ts+++ b/studio/frontend/src/features/recipe-studio/utils/payload/types.ts@@ -69,6 +69,7 @@ export type RecipePayload = {seed_columns?: string[];seed_drop_columns?: string[];seed_preview_rows?: Record<string, unknown>[];+ seed_total_rows?: number;local_file_name?: string;// biome-ignore lint/style/useNamingConvention: api schemaunstructured_upload_uid?: string;diff --git a/studio/frontend/tests/native-training-dataset-drop.test.ts b/studio/frontend/tests/native-training-dataset-drop.test.tsindex f4f3f3b94eb..7e4ea893453 100644--- a/studio/frontend/tests/native-training-dataset-drop.test.ts+++ b/studio/frontend/tests/native-training-dataset-drop.test.ts@@ -181,7 +181,7 @@ test("frontend, backend, and Rust accept the same native dataset extensions", ());});-test("training document redirects match Data Recipes", () => {+test("Data Recipes document types match the backend and cover training redirects", () => {const backendSource = readText("../../backend/routes/data_recipe/seed.py");const recipeSource = readSrc("features/recipe-studio/dialogs/seed/unstructured-drop-zone.tsx",@@ -197,6 +197,20 @@ test("training document redirects match Data Recipes", () => {const training = [...TRAINING_DOCUMENT_REDIRECT_EXTENSIONS].sort();assert.deepEqual(training, [".docx", ".md", ".pdf", ".txt"]);- assert.deepEqual(training, backend);- assert.deepEqual(training, recipe);+ assert.deepEqual(recipe, backend);+ // The Training page redirects only the common types; Data Recipes takes every one it can extract.+ for (const extension of training) {+ assert.ok(backend.includes(extension), extension);+ }+ for (const extension of [+ ".epub",+ ".mobi",+ ".doc",+ ".xlsx",+ ".pptx",+ ".odt",+ ".rtf",+ ]) {+ assert.ok(recipe.includes(extension), extension);+ }});diff --git a/studio/frontend/tests/recipe-seed-total-rows.test.ts b/studio/frontend/tests/recipe-seed-total-rows.test.tsnew file mode 100644index 00000000000..bfebd0ed8d1--- /dev/null+++ b/studio/frontend/tests/recipe-seed-total-rows.test.ts@@ -0,0 +1,54 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import test from "node:test";++import { registerBundlerResolver } from "./helpers/kit.ts";++registerBundlerResolver();++const { importRecipePayload } = await import(+ "../src/features/recipe-studio/utils/import/importer.ts"+);+const { buildRecipePayload } = await import(+ "../src/features/recipe-studio/utils/payload/build-payload.ts"+);++test("a document seed's total chunk count survives save and reload", () => {+ const template = JSON.parse(+ readFileSync(+ new URL(+ "../src/features/data-recipes/learning-recipes/books-and-documents.json",+ import.meta.url,+ ),+ "utf8",+ ),+ );+ template.ui.seed_total_rows = 42;+ const imported = importRecipePayload(JSON.stringify(template));+ assert.ok(imported.snapshot);+ const seed = Object.values(imported.snapshot.configs).find(+ (config) => config.kind === "seed",+ );+ assert.equal(seed?.kind === "seed" ? seed.seed_total_rows : null, 42);++ const {+ configs,+ nodes,+ edges,+ processors,+ layoutDirection,+ auxNodePositions,+ } = imported.snapshot;+ const built = buildRecipePayload(+ configs,+ nodes,+ edges,+ processors,+ layoutDirection,+ auxNodePositions,+ );+ assert.equal(built.payload.ui.seed_total_rows, 42);+});