Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio Data Recipes: keep unstructured seed upload file IO off the event loop (#17266)
Daniel Han1c387c735fad
diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.pyindex 87bc2ff4f0c..f43cebe6845 100644--- a/studio/backend/routes/data_recipe/seed.py+++ b/studio/backend/routes/data_recipe/seed.py@@ -5,6 +5,8 @@from __future__ import annotations+import asyncio+from core.training.account_jobs import account_path, managed_accountimport base64import binascii@@ -18,7 +20,7 @@ from pathlib import Path, PurePosixPathfrom typing import Anyfrom uuid import uuid4-from anyio import CapacityLimiter, to_process, to_thread+from anyio import CancelScope, CapacityLimiter, Lock as _AnyioLock, to_process, to_threadfrom fastapi import APIRouter, Depends, HTTPException, UploadFile, File as FastAPIFile, Formfrom auth.authentication import allow_ambient_hf_token@@ -1202,6 +1204,26 @@ async def _extract_text_from_file_async(file_path: Path, ext: str) -> str:return chunking.normalize_unstructured_text(raw) if chunking is not None else raw+# One upload per block between its budget check and its raw write: the scan, read and write await threads.+_BLOCK_UPLOAD_LOCKS: dict[str, _AnyioLock] = {}+++async def _write_to_completion(write_fn, *args) -> None:+ """A threaded write that, if cancelled, still waits for the thread so cleanup cannot race it."""+ write = asyncio.ensure_future(to_thread.run_sync(write_fn, *args))+ try:+ await asyncio.shield(write)+ except BaseException:+ # Neither a repeated task cancel nor an AnyIO cancel scope may run cleanup before the thread ends.+ with CancelScope(shield = True):+ while not write.done():+ try:+ await asyncio.wait([write])+ except asyncio.CancelledError:+ continue+ raise++def _get_block_total_size(block_dir: Path) -> int:"""Sum raw upload sizes for the whole block from server-owned files."""if not block_dir.exists():@@ -1215,7 +1237,11 @@ def _get_block_total_size(block_dir: Path) -> int:# remove_unstructured_file keys on the name up to the first dot.if is_appledouble_metadata(f):continue- total += f.stat().st_size+ # Runs in a worker thread now, so a delete on the loop can remove a file mid-scan.+ try:+ total += f.stat().st_size+ except FileNotFoundError:+ continuereturn total@@ -1276,22 +1302,16 @@ def _read_native_drop(lease: str, budget: int) -> tuple[str, bytes]:return grant.canonical_path.name, content[email protected]("/seed/upload-unstructured-file")-async def upload_unstructured_file(- file: UploadFile | None = FastAPIFile(None),- block_id: str = Form(...),- native_path_lease: str | None = Form(None, alias = "nativePathLease"),-) -> UnstructuredFileUploadResponse:- _validate_safe_id(block_id, "block_id")-- block_dir = UNSTRUCTURED_UPLOAD_ROOT / block_id+async def _store_unstructured_raw(file, block_dir: Path, native_path_lease):# Reads 0 for a block with no dir yet, without creating it, so an upload about to be refused leaves no directory behind.- budget = UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_BYTES - _get_block_total_size(block_dir)+ budget = UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_BYTES - await to_thread.run_sync(+ _get_block_total_size, block_dir+ )# Tauri drops arrive as a signed path; isinstance since an unfilled Form param is truthy.lease = native_path_lease if isinstance(native_path_lease, str) else Noneif lease:- original_filename, content = _read_native_drop(lease, budget)+ original_filename, content = await to_thread.run_sync(_read_native_drop, lease, budget)elif file is not None and hasattr(file, "read"):original_filename = file.filename or "upload"# Before the read: a rejected 500 MB upload must not be pulled into memory first.@@ -1319,12 +1339,40 @@ async def upload_unstructured_file(file_id = uuid4().hexraw_path = block_dir / f"{file_id}{ext}"- raw_path.write_bytes(content)+ try:+ await _write_to_completion(raw_path.write_bytes, content)+ except BaseException:+ raw_path.unlink(missing_ok = True)+ raise+ return original_filename, content, raw_path, file_id++[email protected]("/seed/upload-unstructured-file")+async def upload_unstructured_file(+ file: UploadFile | None = FastAPIFile(None),+ block_id: str = Form(...),+ native_path_lease: str | None = Form(None, alias = "nativePathLease"),+) -> UnstructuredFileUploadResponse:+ _validate_safe_id(block_id, "block_id")++ block_dir = UNSTRUCTURED_UPLOAD_ROOT / block_id+ lock = _BLOCK_UPLOAD_LOCKS.setdefault(block_id, _AnyioLock())+ try:+ async with lock:+ original_filename, content, raw_path, file_id = await _store_unstructured_raw(+ file, block_dir, native_path_lease+ )+ finally:+ if not lock.locked() and not lock.statistics().tasks_waiting:+ _BLOCK_UPLOAD_LOCKS.pop(block_id, None)+ ext = Path(original_filename).suffix.lower()+ size_bytes = len(content)extracted_path = block_dir / f"{file_id}.extracted.txt"try:extracted_text = await _extract_text_from_file_async(raw_path, ext)- if not extracted_text or not extracted_text.strip():+ # isspace() == (not strip()) on a non-empty str, without copying a 500 MB text on the event loop.+ if not extracted_text or extracted_text.isspace():raw_path.unlink(missing_ok = True)return UnstructuredFileUploadResponse(file_id = file_id,@@ -1333,7 +1381,9 @@ async def upload_unstructured_file(status = "error",error = "No extractable text found in file",)- extracted_path.write_text(extracted_text, encoding = "utf-8")+ await _write_to_completion(+ functools.partial(extracted_path.write_text, extracted_text, encoding = "utf-8")+ )except ImportError as e:raw_path.unlink(missing_ok = True)extracted_path.unlink(missing_ok = True)diff --git a/studio/backend/tests/test_data_recipe_seed.py b/studio/backend/tests/test_data_recipe_seed.pyindex 63f60888213..1871cc0f52c 100644--- a/studio/backend/tests/test_data_recipe_seed.py+++ b/studio/backend/tests/test_data_recipe_seed.py@@ -1895,5 +1895,160 @@ def test_document_extraction_threads_are_bounded(monkeypatch, tmp_path):monkeypatch.setattr(seed_route.to_thread, "run_sync", run_sync)assert _run_upload(seed_route, "notes.txt", b"hello").status == "ok"- assert seen == [seed_route._document_extraction_limiter]+ assert [limiter for limiter in seen if limiter is not None] == [+ seed_route._document_extraction_limiter+ ]assert seed_route._document_extraction_limiter.total_tokens == 2+++def test_concurrent_native_drops_to_one_block_stay_within_its_budget(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ dropped = tmp_path / "notes.txt"+ dropped.write_bytes(b"some extractable text " * 3)+ monkeypatch.setattr(seed_route, "UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_BYTES", 100)+ monkeypatch.setitem(+ sys.modules,+ "utils.native_path_leases",+ SimpleNamespace(+ NativePathLeaseError = RuntimeError,+ verify_native_path_lease = lambda *a, **k: SimpleNamespace(canonical_path = dropped),+ ),+ )++ async def both():+ return await asyncio.gather(+ *(+ seed_route.upload_unstructured_file(None, "block", native_path_lease = "lease")+ for _ in range(2)+ ),+ return_exceptions = True,+ )++ results = asyncio.run(both())+ refused = [r for r in results if isinstance(r, HTTPException)]+ assert len(refused) == 1 and refused[0].status_code == 413+ raw = [n for n in _block_files(seed_route) if n.endswith(".txt") and ".extracted" not in n]+ assert len(raw) == 1+++def test_a_cancelled_raw_write_leaves_no_file(monkeypatch, tmp_path):+ import threading+ import time++ seed_route = _load_seed_route(monkeypatch, tmp_path)+ started = threading.Event()+ real_write = Path.write_bytes++ def slow_write(self, data):+ started.set()+ time.sleep(0.3)+ return real_write(self, data)++ monkeypatch.setattr(Path, "write_bytes", slow_write)++ async def cancel_mid_write():+ task = asyncio.ensure_future(+ seed_route.upload_unstructured_file(_FakeUpload("notes.txt", b"text to keep"), "block")+ )+ while not started.is_set():+ await asyncio.sleep(0.01)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task++ asyncio.run(cancel_mid_write())+ time.sleep(0.4)+ assert _block_files(seed_route) == []+++def test_a_cancelled_extracted_text_write_leaves_no_file(monkeypatch, tmp_path):+ import threading+ import time++ seed_route = _load_seed_route(monkeypatch, tmp_path)+ started = threading.Event()+ real_write = Path.write_text++ def slow_write(self, *args, **kwargs):+ started.set()+ time.sleep(0.3)+ return real_write(self, *args, **kwargs)++ monkeypatch.setattr(Path, "write_text", slow_write)++ async def cancel_mid_write():+ task = asyncio.ensure_future(+ seed_route.upload_unstructured_file(_FakeUpload("notes.txt", b"text to keep"), "block")+ )+ while not started.is_set():+ await asyncio.sleep(0.01)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task++ asyncio.run(cancel_mid_write())+ time.sleep(0.4)+ assert _block_files(seed_route) == []++[email protected]("slow", ["write_bytes", "write_text"])+def test_an_anyio_cancel_scope_leaves_no_file(monkeypatch, tmp_path, slow):+ import time++ import anyio++ seed_route = _load_seed_route(monkeypatch, tmp_path)+ real = getattr(Path, slow)++ def slow_write(self, *args, **kwargs):+ time.sleep(0.3)+ return real(self, *args, **kwargs)++ monkeypatch.setattr(Path, slow, slow_write)++ async def cancel_by_scope():+ with anyio.move_on_after(0.1):+ await seed_route.upload_unstructured_file(+ _FakeUpload("notes.txt", b"text to keep"), "block"+ )++ asyncio.run(cancel_by_scope())+ time.sleep(0.4)+ assert _block_files(seed_route) == []+++def test_a_slow_raw_write_does_not_block_the_event_loop(monkeypatch, tmp_path):+ import time++ seed_route = _load_seed_route(monkeypatch, tmp_path)+ real_write = Path.write_bytes++ def slow_write(self, data):+ time.sleep(0.3)+ return real_write(self, data)++ monkeypatch.setattr(Path, "write_bytes", slow_write)++ async def upload_while_ticking():+ ticks = 0+ done = False++ async def ticker():+ nonlocal ticks+ while not done:+ ticks += 1+ await asyncio.sleep(0.01)++ tick_task = asyncio.ensure_future(ticker())+ await asyncio.sleep(0)+ result = await seed_route.upload_unstructured_file(+ _FakeUpload("notes.txt", b"text to keep"), "block"+ )+ done = True+ await tick_task+ return result, ticks++ result, ticks = asyncio.run(upload_while_ticking())+ assert result.status == "ok"+ assert ticks >= 10+ assert seed_route._BLOCK_UPLOAD_LOCKS == {}