Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio Data Recipes: keep unstructured seed upload file IO off the event loop (#17266)

Daniel Han1c387c735fad
diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.py
index 87bc2ff4f0c..f43cebe6845 100644
--- a/studio/backend/routes/data_recipe/seed.py
+++ b/studio/backend/routes/data_recipe/seed.py
@@ -5,6 +5,8 @@
from __future__ import annotations
+import asyncio
+
from core.training.account_jobs import account_path, managed_account
import base64
import binascii
@@ -18,7 +20,7 @@ from pathlib import Path, PurePosixPath
from typing import Any
from uuid import uuid4
-from anyio import CapacityLimiter, to_process, to_thread
+from anyio import CancelScope, CapacityLimiter, Lock as _AnyioLock, to_process, to_thread
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File as FastAPIFile, Form
from auth.authentication import allow_ambient_hf_token
@@ -1202,6 +1204,26 @@ async def _extract_text_from_file_async(file_path: Path, ext: str) -> str:
return chunking.normalize_unstructured_text(raw) if chunking is not None else raw
+# One upload per block between its budget check and its raw write: the scan, read and write await threads.
+_BLOCK_UPLOAD_LOCKS: dict[str, _AnyioLock] = {}
+
+
+async def _write_to_completion(write_fn, *args) -> None:
+ """A threaded write that, if cancelled, still waits for the thread so cleanup cannot race it."""
+ write = asyncio.ensure_future(to_thread.run_sync(write_fn, *args))
+ try:
+ await asyncio.shield(write)
+ except BaseException:
+ # Neither a repeated task cancel nor an AnyIO cancel scope may run cleanup before the thread ends.
+ with CancelScope(shield = True):
+ while not write.done():
+ try:
+ await asyncio.wait([write])
+ except asyncio.CancelledError:
+ continue
+ raise
+
+
def _get_block_total_size(block_dir: Path) -> int:
"""Sum raw upload sizes for the whole block from server-owned files."""
if not block_dir.exists():
@@ -1215,7 +1237,11 @@ def _get_block_total_size(block_dir: Path) -> int:
# remove_unstructured_file keys on the name up to the first dot.
if is_appledouble_metadata(f):
continue
- total += f.stat().st_size
+ # Runs in a worker thread now, so a delete on the loop can remove a file mid-scan.
+ try:
+ total += f.stat().st_size
+ except FileNotFoundError:
+ continue
return total
@@ -1276,22 +1302,16 @@ def _read_native_drop(lease: str, budget: int) -> tuple[str, bytes]:
return grant.canonical_path.name, content
[email protected]("/seed/upload-unstructured-file")
-async def upload_unstructured_file(
- file: UploadFile | None = FastAPIFile(None),
- block_id: str = Form(...),
- native_path_lease: str | None = Form(None, alias = "nativePathLease"),
-) -> UnstructuredFileUploadResponse:
- _validate_safe_id(block_id, "block_id")
-
- block_dir = UNSTRUCTURED_UPLOAD_ROOT / block_id
+async def _store_unstructured_raw(file, block_dir: Path, native_path_lease):
# Reads 0 for a block with no dir yet, without creating it, so an upload about to be refused leaves no directory behind.
- budget = UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_BYTES - _get_block_total_size(block_dir)
+ budget = UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_BYTES - await to_thread.run_sync(
+ _get_block_total_size, block_dir
+ )
# Tauri drops arrive as a signed path; isinstance since an unfilled Form param is truthy.
lease = native_path_lease if isinstance(native_path_lease, str) else None
if lease:
- original_filename, content = _read_native_drop(lease, budget)
+ original_filename, content = await to_thread.run_sync(_read_native_drop, lease, budget)
elif file is not None and hasattr(file, "read"):
original_filename = file.filename or "upload"
# Before the read: a rejected 500 MB upload must not be pulled into memory first.
@@ -1319,12 +1339,40 @@ async def upload_unstructured_file(
file_id = uuid4().hex
raw_path = block_dir / f"{file_id}{ext}"
- raw_path.write_bytes(content)
+ try:
+ await _write_to_completion(raw_path.write_bytes, content)
+ except BaseException:
+ raw_path.unlink(missing_ok = True)
+ raise
+ return original_filename, content, raw_path, file_id
+
+
[email protected]("/seed/upload-unstructured-file")
+async def upload_unstructured_file(
+ file: UploadFile | None = FastAPIFile(None),
+ block_id: str = Form(...),
+ native_path_lease: str | None = Form(None, alias = "nativePathLease"),
+) -> UnstructuredFileUploadResponse:
+ _validate_safe_id(block_id, "block_id")
+
+ block_dir = UNSTRUCTURED_UPLOAD_ROOT / block_id
+ lock = _BLOCK_UPLOAD_LOCKS.setdefault(block_id, _AnyioLock())
+ try:
+ async with lock:
+ original_filename, content, raw_path, file_id = await _store_unstructured_raw(
+ file, block_dir, native_path_lease
+ )
+ finally:
+ if not lock.locked() and not lock.statistics().tasks_waiting:
+ _BLOCK_UPLOAD_LOCKS.pop(block_id, None)
+ ext = Path(original_filename).suffix.lower()
+ size_bytes = len(content)
extracted_path = block_dir / f"{file_id}.extracted.txt"
try:
extracted_text = await _extract_text_from_file_async(raw_path, ext)
- if not extracted_text or not extracted_text.strip():
+ # isspace() == (not strip()) on a non-empty str, without copying a 500 MB text on the event loop.
+ if not extracted_text or extracted_text.isspace():
raw_path.unlink(missing_ok = True)
return UnstructuredFileUploadResponse(
file_id = file_id,
@@ -1333,7 +1381,9 @@ async def upload_unstructured_file(
status = "error",
error = "No extractable text found in file",
)
- extracted_path.write_text(extracted_text, encoding = "utf-8")
+ await _write_to_completion(
+ functools.partial(extracted_path.write_text, extracted_text, encoding = "utf-8")
+ )
except ImportError as e:
raw_path.unlink(missing_ok = True)
extracted_path.unlink(missing_ok = True)
diff --git a/studio/backend/tests/test_data_recipe_seed.py b/studio/backend/tests/test_data_recipe_seed.py
index 63f60888213..1871cc0f52c 100644
--- a/studio/backend/tests/test_data_recipe_seed.py
+++ b/studio/backend/tests/test_data_recipe_seed.py
@@ -1895,5 +1895,160 @@ def test_document_extraction_threads_are_bounded(monkeypatch, tmp_path):
monkeypatch.setattr(seed_route.to_thread, "run_sync", run_sync)
assert _run_upload(seed_route, "notes.txt", b"hello").status == "ok"
- assert seen == [seed_route._document_extraction_limiter]
+ assert [limiter for limiter in seen if limiter is not None] == [
+ seed_route._document_extraction_limiter
+ ]
assert seed_route._document_extraction_limiter.total_tokens == 2
+
+
+def test_concurrent_native_drops_to_one_block_stay_within_its_budget(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ dropped = tmp_path / "notes.txt"
+ dropped.write_bytes(b"some extractable text " * 3)
+ monkeypatch.setattr(seed_route, "UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_BYTES", 100)
+ monkeypatch.setitem(
+ sys.modules,
+ "utils.native_path_leases",
+ SimpleNamespace(
+ NativePathLeaseError = RuntimeError,
+ verify_native_path_lease = lambda *a, **k: SimpleNamespace(canonical_path = dropped),
+ ),
+ )
+
+ async def both():
+ return await asyncio.gather(
+ *(
+ seed_route.upload_unstructured_file(None, "block", native_path_lease = "lease")
+ for _ in range(2)
+ ),
+ return_exceptions = True,
+ )
+
+ results = asyncio.run(both())
+ refused = [r for r in results if isinstance(r, HTTPException)]
+ assert len(refused) == 1 and refused[0].status_code == 413
+ raw = [n for n in _block_files(seed_route) if n.endswith(".txt") and ".extracted" not in n]
+ assert len(raw) == 1
+
+
+def test_a_cancelled_raw_write_leaves_no_file(monkeypatch, tmp_path):
+ import threading
+ import time
+
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ started = threading.Event()
+ real_write = Path.write_bytes
+
+ def slow_write(self, data):
+ started.set()
+ time.sleep(0.3)
+ return real_write(self, data)
+
+ monkeypatch.setattr(Path, "write_bytes", slow_write)
+
+ async def cancel_mid_write():
+ task = asyncio.ensure_future(
+ seed_route.upload_unstructured_file(_FakeUpload("notes.txt", b"text to keep"), "block")
+ )
+ while not started.is_set():
+ await asyncio.sleep(0.01)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ asyncio.run(cancel_mid_write())
+ time.sleep(0.4)
+ assert _block_files(seed_route) == []
+
+
+def test_a_cancelled_extracted_text_write_leaves_no_file(monkeypatch, tmp_path):
+ import threading
+ import time
+
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ started = threading.Event()
+ real_write = Path.write_text
+
+ def slow_write(self, *args, **kwargs):
+ started.set()
+ time.sleep(0.3)
+ return real_write(self, *args, **kwargs)
+
+ monkeypatch.setattr(Path, "write_text", slow_write)
+
+ async def cancel_mid_write():
+ task = asyncio.ensure_future(
+ seed_route.upload_unstructured_file(_FakeUpload("notes.txt", b"text to keep"), "block")
+ )
+ while not started.is_set():
+ await asyncio.sleep(0.01)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ asyncio.run(cancel_mid_write())
+ time.sleep(0.4)
+ assert _block_files(seed_route) == []
+
+
[email protected]("slow", ["write_bytes", "write_text"])
+def test_an_anyio_cancel_scope_leaves_no_file(monkeypatch, tmp_path, slow):
+ import time
+
+ import anyio
+
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ real = getattr(Path, slow)
+
+ def slow_write(self, *args, **kwargs):
+ time.sleep(0.3)
+ return real(self, *args, **kwargs)
+
+ monkeypatch.setattr(Path, slow, slow_write)
+
+ async def cancel_by_scope():
+ with anyio.move_on_after(0.1):
+ await seed_route.upload_unstructured_file(
+ _FakeUpload("notes.txt", b"text to keep"), "block"
+ )
+
+ asyncio.run(cancel_by_scope())
+ time.sleep(0.4)
+ assert _block_files(seed_route) == []
+
+
+def test_a_slow_raw_write_does_not_block_the_event_loop(monkeypatch, tmp_path):
+ import time
+
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ real_write = Path.write_bytes
+
+ def slow_write(self, data):
+ time.sleep(0.3)
+ return real_write(self, data)
+
+ monkeypatch.setattr(Path, "write_bytes", slow_write)
+
+ async def upload_while_ticking():
+ ticks = 0
+ done = False
+
+ async def ticker():
+ nonlocal ticks
+ while not done:
+ ticks += 1
+ await asyncio.sleep(0.01)
+
+ tick_task = asyncio.ensure_future(ticker())
+ await asyncio.sleep(0)
+ result = await seed_route.upload_unstructured_file(
+ _FakeUpload("notes.txt", b"text to keep"), "block"
+ )
+ done = True
+ await tick_task
+ return result, ticks
+
+ result, ticks = asyncio.run(upload_while_ticking())
+ assert result.status == "ok"
+ assert ticks >= 10
+ assert seed_route._BLOCK_UPLOAD_LOCKS == {}