Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fix document upload stalls, races and text decoding (#10496)

Michael Hancedbb58e4a49
diff --git a/.github/workflows/rag-upload-compatibility.yml b/.github/workflows/rag-upload-compatibility.yml
new file mode 100644
index 00000000000..55b0006d4be
--- /dev/null
+++ b/.github/workflows/rag-upload-compatibility.yml
@@ -0,0 +1,49 @@
+name: Document upload compatibility
+
+on:
+ pull_request:
+ paths:
+ - "studio/backend/core/rag/**"
+ - "studio/backend/tests/test_rag*"
+ - "studio/frontend/src/features/rag/**"
+ - "studio/frontend/tests/rag-*"
+ - "tests/studio/rag_upload*"
+ - ".github/workflows/rag-upload-compatibility.yml"
+
+permissions:
+ contents: read
+
+concurrency:
+ group: rag-upload-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ compatibility:
+ name: Uploads (${{ matrix.os }})
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 20
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, macos-latest, windows-latest]
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+ with:
+ persist-credentials: false
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
+ with:
+ python-version: "3.12"
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
+ with:
+ node-version: "24"
+ - name: Run isolated simulations
+ run: python tests/studio/rag_upload_ci.py
+ - name: Upload results
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
+ with:
+ name: document-upload-${{ matrix.os }}
+ path: |
+ temp/rag-upload/*.log
+ temp/rag-upload/*.json
+ temp/rag-upload/*.xml
diff --git a/studio/backend/core/rag/captioner.py b/studio/backend/core/rag/captioner.py
index b6c301c03c2..af850b008fd 100644
--- a/studio/backend/core/rag/captioner.py
+++ b/studio/backend/core/rag/captioner.py
@@ -13,6 +13,7 @@ from __future__ import annotations
import base64
import contextlib
import logging
+from collections.abc import Callable
from . import config
@@ -174,7 +175,10 @@ def _ocr_one(base_url: str, model: str, image_bytes: bytes, timeout: float) -> s
def caption_images(
- images: list, *, endpoint: tuple[str, str] | None = None
+ images: list,
+ *,
+ endpoint: tuple[str, str] | None = None,
+ on_progress: Callable[[int, int], None] | None = None,
) -> dict[int, list[str]]:
"""Caption ``ParsedImage`` objects, keyed by 1-based page number; ``{}`` when there
are no images or no vision model. The caller (`ingestion._run`) owns the on/off
@@ -187,19 +191,24 @@ def caption_images(
base_url, model = ep
out: dict[int, list[str]] = {}
- for img in images[: config.CAPTION_MAX_IMAGES]:
- image_bytes = getattr(img, "image_bytes", None)
- if not image_bytes:
- continue
- caption = _caption_one(base_url, model, image_bytes, config.CAPTION_TIMEOUT_S)
+ selected = [
+ img for img in images[: config.CAPTION_MAX_IMAGES] if getattr(img, "image_bytes", None)
+ ]
+ for index, img in enumerate(selected, 1):
+ caption = _caption_one(base_url, model, img.image_bytes, config.CAPTION_TIMEOUT_S)
if caption:
page = getattr(img, "page_number", None) or 0
out.setdefault(int(page), []).append(_collapse_runaway(caption))
+ if on_progress is not None:
+ on_progress(index, len(selected))
return out
def ocr_pages(
- page_pngs: dict[int, bytes], *, endpoint: tuple[str, str] | None = None
+ page_pngs: dict[int, bytes],
+ *,
+ endpoint: tuple[str, str] | None = None,
+ on_progress: Callable[[int, int], None] | None = None,
) -> dict[int, str]:
"""OCR rendered page PNGs (keyed by 1-based page number) to text; ``{}`` when there
is no vision model or no pages. The caller (`ingestion._ocr_scanned_pages`) owns the
@@ -212,10 +221,13 @@ def ocr_pages(
base_url, model = ep
out: dict[int, str] = {}
- for page_num in sorted(page_pngs)[: config.OCR_MAX_PAGES]:
+ selected = sorted(page_pngs)[: config.OCR_MAX_PAGES]
+ for index, page_num in enumerate(selected, 1):
text = _ocr_one(base_url, model, page_pngs[page_num], config.OCR_TIMEOUT_S)
if text:
out[int(page_num)] = _collapse_runaway(text)
+ if on_progress is not None:
+ on_progress(index, len(selected))
return out
diff --git a/studio/backend/core/rag/chunking.py b/studio/backend/core/rag/chunking.py
index cbb4530b1ad..3200f87aa74 100644
--- a/studio/backend/core/rag/chunking.py
+++ b/studio/backend/core/rag/chunking.py
@@ -27,40 +27,59 @@ class Chunk:
page_char_end: int
-def _split(text: str, seps: tuple[str, ...], max_tokens: int, count: TokenCounter) -> list[str]:
+@dataclass(frozen = True, slots = True)
+class _Piece:
+ text: str
+ token_count: int
+
+
+def _split(
+ text: str,
+ seps: tuple[str, ...],
+ max_tokens: int,
+ count: TokenCounter,
+ token_count: int | None = None,
+) -> list[_Piece]:
"""Recursively split into pieces each <= max_tokens (best effort). Pieces
rejoin to ``text`` exactly, so offsets are a running length."""
- if count(text) <= max_tokens:
- return [text]
+ if token_count is None:
+ token_count = count(text)
+ if token_count <= max_tokens:
+ return [_Piece(text, token_count)]
for i, sep in enumerate(seps):
parts = list(text) if sep == "" else text.split(sep)
if len(parts) <= 1:
continue
if sep:
parts = [p + sep for p in parts[:-1]] + parts[-1:]
- out: list[str] = []
+ out: list[_Piece] = []
for p in parts:
+ if not p:
+ continue
+ tokens = count(p)
out.extend(
- [p] if count(p) <= max_tokens else _split(p, seps[i + 1 :], max_tokens, count)
+ [_Piece(p, tokens)]
+ if tokens <= max_tokens
+ else _split(p, seps[i + 1 :], max_tokens, count, tokens)
)
- return [p for p in out if p]
+ return out
n = max(1, max_tokens * 4)
- return [text[j : j + n] for j in range(0, len(text), n)]
+ return [_Piece(part, count(part)) for j in range(0, len(text), n) if (part := text[j : j + n])]
def _merge(
- pieces: list[str], starts: list[int], max_tokens: int, overlap: int, count: TokenCounter
+ pieces: list[_Piece], starts: list[int], max_tokens: int, overlap: int
) -> list[tuple[str, int, int]]:
"""Greedy-merge pieces into <= max_tokens chunks with token overlap.
``starts[i]`` is ``pieces[i]``'s page char offset; returns
``(chunk_text, char_start, char_end)`` spans."""
chunks: list[tuple[str, int, int]] = []
- buf: list[str] = []
+ buf: list[_Piece] = []
buf_starts: list[int] = []
buf_tok = 0
def _flush() -> None:
- raw = "".join(buf)
+ raw = "".join(piece.text for piece in buf)
stripped = raw.strip()
if not stripped:
return
@@ -71,7 +90,8 @@ def _merge(
chunks.append((stripped, start, end))
for piece, start in zip(pieces, starts):
- pt = count(piece)
+ # Reuse counts after the GGUF tokenizer's cache evicts earlier pieces.
+ pt = piece.token_count
if buf and buf_tok + pt > max_tokens:
_flush()
# Bound the carry so carry + this piece fits max_tokens; else a full overlap before a near-max piece
@@ -79,11 +99,11 @@ def _merge(
carry_budget = min(overlap, max(0, max_tokens - pt))
carry, carry_starts, run = [], [], 0
for prev, prev_start in zip(reversed(buf), reversed(buf_starts)):
- if run + count(prev) > carry_budget:
+ if run + prev.token_count > carry_budget:
break
carry.insert(0, prev)
carry_starts.insert(0, prev_start)
- run += count(prev)
+ run += prev.token_count
buf, buf_starts, buf_tok = carry, carry_starts, run
buf.append(piece)
buf_starts.append(start)
@@ -105,8 +125,8 @@ def chunk_pages(
cursor = 0
for piece in pieces:
starts.append(cursor)
- cursor += len(piece)
- for text, char_start, char_end in _merge(pieces, starts, max_tokens, overlap, count):
+ cursor += len(piece.text)
+ for text, char_start, char_end in _merge(pieces, starts, max_tokens, overlap):
out.append(
Chunk(
text = text,
diff --git a/studio/backend/core/rag/config.py b/studio/backend/core/rag/config.py
index cac9c4201ff..ecc91c40027 100644
--- a/studio/backend/core/rag/config.py
+++ b/studio/backend/core/rag/config.py
@@ -51,8 +51,8 @@ FOLDER_JOB_HISTORY_LIMIT = int(os.environ.get("RAG_FOLDER_JOB_HISTORY_LIMIT", "2
# Falls back to plain PyMuPDF text when off, when pymupdf4llm is missing, or when extraction fails.
PDF_MARKDOWN = os.environ.get("RAG_PDF_MARKDOWN", "1") == "1"
-# No-op without a vision model; the chat's "Describe figures & charts" toggle overrides it per upload.
-CAPTION_IMAGES = os.environ.get("RAG_CAPTION_IMAGES", "1") == "1"
+# Figure descriptions are opt-in; the chat toggle overrides this default.
+CAPTION_IMAGES = os.environ.get("RAG_CAPTION_IMAGES", "0") == "1"
# Total per-document tile budget (figure-bearing pages are tiled, see below).
CAPTION_MAX_IMAGES = int(os.environ.get("RAG_CAPTION_MAX_IMAGES", "24"))
CAPTION_TIMEOUT_S = float(os.environ.get("RAG_CAPTION_TIMEOUT_S", "60"))
diff --git a/studio/backend/core/rag/ingestion.py b/studio/backend/core/rag/ingestion.py
index 3dc31bc397e..26fe5fc86df 100644
--- a/studio/backend/core/rag/ingestion.py
+++ b/studio/backend/core/rag/ingestion.py
@@ -14,6 +14,7 @@ import os
import queue
import re
import threading
+from collections.abc import Callable
from storage import rag_db
@@ -47,8 +48,15 @@ def _remove_upload(stored_path: str | None, *, keep_path: str | None = None) ->
return
try:
target = os.path.realpath(stored_path)
- if keep_path is not None and target == os.path.realpath(keep_path):
- return
+ if keep_path is not None:
+ if target == os.path.realpath(keep_path):
+ return
+ # Case aliases can name the same file on macOS and Windows.
+ try:
+ if os.path.samefile(target, keep_path):
+ return
+ except OSError:
+ pass
from utils.paths import rag_uploads_root
uploads = os.path.realpath(str(rag_uploads_root()))
@@ -110,7 +118,11 @@ def _abort_if_document_deleted(conn, job_id: str, document_id: str) -> bool:
return True
-def _embed_pass(texts: list[str], model_name: str | None):
+def _embed_pass(
+ texts: list[str],
+ model_name: str | None,
+ on_progress: Callable[[int, int], None] | None = None,
+):
"""One batched pass. Returns ``(vectors, identity, changed)``, ``changed`` when the
embedder swapped part way and the vectors therefore span two spaces."""
vectors: list = []
@@ -124,17 +136,23 @@ def _embed_pass(texts: list[str], model_name: str | None):
changed = changed or (identity is not None and batch_identity != identity)
identity = batch_identity
vectors.extend(out)
+ if on_progress is not None:
+ on_progress(min(i + _EMBED_BATCH, len(texts)), len(texts))
return vectors, identity or embeddings.embedding_identity(model_name), changed
-def _embed_all(texts: list[str], model_name: str | None):
+def _embed_all(
+ texts: list[str],
+ model_name: str | None,
+ on_progress: Callable[[int, int], None] | None = None,
+):
"""Embed texts in batches. Returns ``(vectors, identity)`` of the embedder that
produced them. An ST encode failure swaps the process to llama-server, and a swap
between batches would leave one document holding vectors from two spaces, so the
document restarts under the backend that took over. That swap is one-way, so the
second pass is uniform."""
for _ in range(2):
- vectors, identity, changed = _embed_pass(texts, model_name)
+ vectors, identity, changed = _embed_pass(texts, model_name, on_progress)
if not changed:
return vectors, identity
logger.warning("embedder changed mid-document; re-embedding under the new one")
@@ -173,7 +191,10 @@ def _ocr_scanned_pages(
scanned = scanned[: config.OCR_MAX_PAGES]
_progress(conn, job_id, "ocr", 0.25)
page_pngs = parsers.render_pdf_pages(stored_path, scanned, dpi = config.OCR_DPI)
- texts = captioner.ocr_pages(page_pngs)
+ texts = captioner.ocr_pages(
+ page_pngs,
+ on_progress = lambda done, total: _progress(conn, job_id, "ocr", 0.25 + 0.15 * done / total),
+ )
if not texts:
return pages, set()
@@ -217,6 +238,38 @@ def _replace_old_document(
logger.warning("failed to remove replaced document %s", old_id, exc_info = True)
+def _retire_orphan_after_failure(
+ conn, replaces: tuple[str, str | None] | None, keep_path: str, document_id: str
+) -> None:
+ """Clear a never-indexed document whose replacement did not complete.
+
+ Orphans only: an ``empty_completed`` / stale-embedder original is still searchable. The
+ orphan has no live job, so startup repair (which scans jobs) never reaches it and the
+ scope would stay indexing forever, holding queued chat sends.
+
+ Checked against the replacement inside the transaction, as ``_replace_old_document`` is: a
+ delete that removed the replacement took its upload too, so dropping the orphan there would
+ discard the last copy. Failing that one instead still frees the scope.
+ """
+ if replaces is None:
+ return
+ old_id, old_path = replaces
+ try:
+ conn.execute("BEGIN IMMEDIATE")
+ doc = store.get_document(conn, old_id)
+ if doc is None or doc.get("status") not in {"pending", "running"}:
+ conn.rollback()
+ return
+ if store.get_document(conn, document_id) is None:
+ conn.rollback()
+ store.set_document_status(conn, old_id, "failed", error = "Indexing did not finish")
+ return
+ store.delete_document(conn, old_id)
+ _remove_upload(old_path, keep_path = keep_path)
+ except Exception: # noqa: BLE001 - cleanup must not mask the original failure
+ logger.warning("failed to retire orphaned document %s", old_id, exc_info = True)
+
+
def _run(
job_id: str,
document_id: str,
@@ -239,6 +292,7 @@ def _run(
caption_on = config.CAPTION_IMAGES if caption is None else caption
# Skip all figure work (PDF rasterization included) without a vision model.
if caption_on and is_pdf and captioner.vision_endpoint() is not None:
+ _progress(conn, job_id, "captioning", 0.4)
# Tile figure pages, transcribe+describe each tile, then merge/dedup/splice into the page text so
# small labels and every sub-figure are captured.
try:
@@ -266,11 +320,17 @@ def _run(
logger.warning("figure tiling failed for job %s", job_id, exc_info = True)
tiles = []
if tiles:
- _progress(conn, job_id, "captioning", 0.28)
- captions = captioner.merge_page_captions(captioner.caption_images(tiles))
+ captions = captioner.merge_page_captions(
+ captioner.caption_images(
+ tiles,
+ on_progress = lambda done, total: _progress(
+ conn, job_id, "captioning", 0.4 + 0.2 * done / total
+ ),
+ )
+ )
pages = captioner.splice_captions(pages, captions)
- _progress(conn, job_id, "chunking", 0.3)
+ _progress(conn, job_id, "chunking", 0.6)
count = embeddings.token_counter(model_name)
chunks = chunking.chunk_pages(
pages,
@@ -292,10 +352,18 @@ def _run(
_emit(job_id, {"type": "complete", "num_chunks": 0})
return
- _progress(conn, job_id, "embedding", 0.5)
+ _progress(conn, job_id, "embedding", 0.65)
# An ST encode failure swaps the process to llama-server, so the embedder that produced these
# vectors is only known once they exist.
- vectors, identity = _embed_all([c.text for c in chunks], model_name)
+ embedded_progress = 0.65
+
+ def report_embeddings(done, total):
+ nonlocal embedded_progress
+ # Keep progress monotonic when a backend swap repeats a batch.
+ embedded_progress = max(embedded_progress, 0.65 + 0.25 * done / total)
+ _progress(conn, job_id, "embedding", embedded_progress)
+
+ vectors, identity = _embed_all([c.text for c in chunks], model_name, report_embeddings)
store.set_document_embedding_model(conn, document_id, identity)
# Locate each chunk's highlight regions (non-PDFs/failures yield none).
@@ -308,7 +376,7 @@ def _run(
logger.warning("pdf region location failed for job %s", job_id, exc_info = True)
regions = None
- _progress(conn, job_id, "storing", 0.9)
+ _progress(conn, job_id, "storing", 0.95)
if _abort_if_document_deleted(conn, job_id, document_id):
return
store.add_chunks(conn, scope, document_id, chunks, vectors, regions)
@@ -335,6 +403,9 @@ def _run(
_emit(job_id, {"type": "error", "stage": "error", "error": str(exc)})
finally:
if conn is not None:
+ # Every exit but a completed one, which already retired its orphan. Nothing relaunches
+ # ingestion, so a lost lease ends the work too: only _new_job ever claims one.
+ _retire_orphan_after_failure(conn, replaces, stored_path, document_id)
conn.close()
job_leases.release(job_leases.INGESTION, job_id)
with _jobs_lock:
@@ -361,7 +432,8 @@ def start_ingestion(
) -> tuple[str, str]:
"""Create the document + job rows and spawn the worker, returning
``(document_id, job_id)``. A duplicate content hash in this scope returns the
- existing id with an already-completed job (no re-ingest).
+ existing id and its active job while indexing, or an already-completed job
+ when the document is ready (no re-ingest).
``content_hash`` lets a caller that already hashed ``stored_path`` (linked-folder
reconciliation hashes it to detect content-identical renames) pass that digest
@@ -398,6 +470,17 @@ def start_ingestion(
existing = store.document_by_hash(conn, scope, sha) if dedupe else None
if existing is not None:
doc = store.get_document(conn, existing)
+ in_progress = doc.get("status") in {"pending", "running"}
+ if in_progress:
+ job = conn.execute(
+ "SELECT id FROM ingestion_jobs WHERE document_id=? "
+ "AND status IN ('pending','running') ORDER BY created_at DESC LIMIT 1",
+ (existing,),
+ ).fetchone()
+ if job is not None:
+ conn.commit()
+ _remove_upload(stored_path, keep_path = doc.get("stored_path"))
+ return existing, job["id"]
empty_completed = (
doc is not None and doc.get("status") == "completed" and not doc.get("num_chunks")
)
@@ -411,12 +494,12 @@ def start_ingestion(
doc.get("embedding_model"), effective_identity
)
)
- if empty_completed or stale_model:
- # Zero chunks previously, or a different embedder: re-ingest rather than dedupe.
+ if empty_completed or stale_model or in_progress:
+ # Retry empty, stale or orphaned documents; keep the old copy until success.
replaces = (existing, doc.get("stored_path"))
else:
job_id = _new_job(conn, existing, scope, status = "completed", progress = 1.0)
- _remove_upload(stored_path)
+ _remove_upload(stored_path, keep_path = doc.get("stored_path"))
with _jobs_lock:
_jobs[job_id] = queue.Queue()
_emit(
@@ -481,6 +564,12 @@ def start_ingestion(
_workers.pop(job_id, None)
job_leases.release(job_leases.INGESTION, job_id)
fail_stalled_job(job_id, "Ingestion worker could not start")
+ # _run never entered, so its finally cannot retire the orphan this retry replaced.
+ conn = rag_db.get_connection()
+ try:
+ _retire_orphan_after_failure(conn, replaces, stored_path, document_id)
+ finally:
+ conn.close()
raise
return document_id, job_id
diff --git a/studio/backend/core/rag/parsers.py b/studio/backend/core/rag/parsers.py
index caf62b708b7..8b426b3c027 100644
--- a/studio/backend/core/rag/parsers.py
+++ b/studio/backend/core/rag/parsers.py
@@ -10,6 +10,7 @@ are lazy, so importing this module never fails on a missing dep.
from __future__ import annotations
+import codecs
import logging
import os
import re
@@ -447,7 +448,20 @@ def parse(path: str, *, want_images: bool = False):
return (pages, []) if want_images else pages
if ext in (".html", ".htm", ".txt", ".md", ".markdown"):
- with open(path, encoding = "utf-8", errors = "replace") as f:
+ # Honor Unicode BOMs; check UTF-32 before its overlapping UTF-16 prefix.
+ with open(path, "rb") as f:
+ prefix = f.read(4)
+ encoding = "utf-8-sig"
+ for bom, codec in (
+ (codecs.BOM_UTF32_LE, "utf-32"),
+ (codecs.BOM_UTF32_BE, "utf-32"),
+ (codecs.BOM_UTF16_LE, "utf-16"),
+ (codecs.BOM_UTF16_BE, "utf-16"),
+ ):
+ if prefix.startswith(bom):
+ encoding = codec
+ break
+ with open(path, encoding = encoding, errors = "replace") as f:
raw = f.read()
pages = _html(raw) if ext in (".html", ".htm") else [_page(raw, None)]
return (pages, []) if want_images else pages
diff --git a/studio/backend/tests/test_rag_chunking.py b/studio/backend/tests/test_rag_chunking.py
index 3d3c9eedd84..7f37a5d30c4 100644
--- a/studio/backend/tests/test_rag_chunking.py
+++ b/studio/backend/tests/test_rag_chunking.py
@@ -6,6 +6,9 @@
from core.rag.chunking import chunk_pages
from core.rag.parsers import Page, parse_text
+from collections import Counter
+from functools import lru_cache
+
WORDS = lambda t: len(t.split()) # noqa: E731
@@ -67,3 +70,20 @@ def test_parse_text_single_page():
pages = parse_text("hello world")
assert len(pages) == 1
assert pages[0].char_count == len("hello world")
+
+
+def test_large_document_does_not_retokenize_every_piece_after_cache_eviction():
+ # Merging must not repeat HTTP requests after the 4096-entry cache fills.
+ requests = Counter()
+
+ @lru_cache(maxsize = 4096)
+ def count(text):
+ requests[text] += 1
+ return len(text.split())
+
+ text = " ".join(f"word{i}" for i in range(10_000))
+ chunks = chunk_pages([_page(text)], max_tokens = 500, overlap = 64, count = count)
+ assert max(requests.values()) == 1
+ assert len(chunks) > 20
+ assert all(c.token_count <= 500 for c in chunks)
+ assert chunks[-1].page_char_end == len(text)
diff --git a/studio/backend/tests/test_rag_chunking_simulations.py b/studio/backend/tests/test_rag_chunking_simulations.py
new file mode 100644
index 00000000000..e2dc01aa835
--- /dev/null
+++ b/studio/backend/tests/test_rag_chunking_simulations.py
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Generated Unicode and separator cases preserve text and token bounds."""
+
+import pytest
+
+pytest.importorskip("hypothesis")
+from hypothesis import given, settings, strategies as st
+
+from core.rag.chunking import chunk_pages
+from core.rag.parsers import parse_text
+
+
[email protected]("counter", ["words", "characters", "utf8"])
+@settings(max_examples = 250, deadline = None, derandomize = True, database = None)
+@given(
+ text = st.text(alphabet = "abcdef0123 \r\n\t#.| café東京漢字\U0001f600", min_size = 0, max_size = 2000),
+ limit = st.integers(min_value = 8, max_value = 128),
+ overlap = st.integers(min_value = 0, max_value = 64),
+)
+def test_generated_chunks_preserve_every_nonspace_character(counter, text, limit, overlap):
+ counters = {
+ "words": lambda value: len(value.split()),
+ "characters": len,
+ "utf8": lambda value: len(value.encode("utf-8")),
+ }
+ count = counters[counter]
+ chunks = chunk_pages(
+ parse_text(text), max_tokens = limit, overlap = min(overlap, limit - 1), count = count
+ )
+ covered = set()
+ for index, chunk in enumerate(chunks):
+ assert chunk.chunk_index == index
+ assert chunk.text == text[chunk.page_char_start : chunk.page_char_end]
+ assert chunk.token_count == count(chunk.text) <= limit
+ covered.update(range(chunk.page_char_start, chunk.page_char_end))
+ assert all(i in covered for i, char in enumerate(text) if not char.isspace())
diff --git a/studio/backend/tests/test_rag_embedding_identity.py b/studio/backend/tests/test_rag_embedding_identity.py
index c26dae13d7a..bec168057e5 100644
--- a/studio/backend/tests/test_rag_embedding_identity.py
+++ b/studio/backend/tests/test_rag_embedding_identity.py
@@ -386,7 +386,11 @@ def test_a_swap_between_batches_re_embeds_the_document(
passes = {"n": 0}
real_pass = ingestion._embed_pass
- def swap_during_the_first_pass(texts, model_name):
+ def swap_during_the_first_pass(
+ texts,
+ model_name,
+ on_progress = None,
+ ):
passes["n"] += 1
if passes["n"] == 1:
calls = {"n": 0}
@@ -402,7 +406,7 @@ def test_a_swap_between_batches_re_embeds_the_document(
return out
monkeypatch.setattr(embeddings, "encode_with_identity", swap_after_one_batch)
- return real_pass(texts, model_name)
+ return real_pass(texts, model_name, on_progress)
monkeypatch.setattr(ingestion, "_embed_pass", swap_during_the_first_pass)
scope = store.kb_scope("K8")
diff --git a/studio/backend/tests/test_rag_ingestion.py b/studio/backend/tests/test_rag_ingestion.py
index 63a5f220dce..e6af381e64e 100644
--- a/studio/backend/tests/test_rag_ingestion.py
+++ b/studio/backend/tests/test_rag_ingestion.py
@@ -114,8 +114,12 @@ def test_ingestion_skips_chunk_write_when_the_document_was_deleted(
deleted = {}
doc_id_known = threading.Event()
- def delete_document_then_embed(texts, model_name):
- vectors = real_embed_all(texts, model_name)
+ def delete_document_then_embed(
+ texts,
+ model_name,
+ on_progress = None,
+ ):
+ vectors = real_embed_all(texts, model_name, on_progress)
doc_id_known.wait(30)
conn = rag_db.get_connection()
try:
diff --git a/studio/backend/tests/test_rag_ocr_fallback.py b/studio/backend/tests/test_rag_ocr_fallback.py
index c7be1fe60b9..ac70257dc01 100644
--- a/studio/backend/tests/test_rag_ocr_fallback.py
+++ b/studio/backend/tests/test_rag_ocr_fallback.py
@@ -136,7 +136,7 @@ def test_ocr_scanned_pages_merges_short_text_layer(rag_conn, monkeypatch):
monkeypatch.setattr(captioner.config, "OCR_MIN_CHARS", 16)
monkeypatch.setattr(captioner, "vision_endpoint", lambda: ("http://x", "local"))
monkeypatch.setattr(parsers, "render_pdf_pages", lambda *a, **k: {1: b"png"})
- monkeypatch.setattr(captioner, "ocr_pages", lambda page_pngs: {1: "OCR body text"})
+ monkeypatch.setattr(captioner, "ocr_pages", lambda page_pngs, **kwargs: {1: "OCR body text"})
out, ocred = ingestion._ocr_scanned_pages(pages, "scan.pdf", rag_conn, job_id)
assert ocred == {1}
diff --git a/studio/backend/tests/test_rag_upload_formats.py b/studio/backend/tests/test_rag_upload_formats.py
new file mode 100644
index 00000000000..040b9f4c8cf
--- /dev/null
+++ b/studio/backend/tests/test_rag_upload_formats.py
@@ -0,0 +1,182 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Real document bytes through browser/native persistence, parsing and indexing."""
+
+import base64
+import codecs
+from pathlib import Path
+
+import pytest
+from fastapi import HTTPException, UploadFile
+
+from core.rag import captioner, config, ingestion, parsers, store
+from routes.rag import _resolve_document_upload
+from storage import rag_db
+from .test_rag_native_drop_upload import SECRET, _sign
+import utils.native_path_leases as leases
+
+EXTENSIONS = (".pdf", ".txt", ".md", ".markdown", ".docx", ".html", ".htm")
+TEXT_EXTENSIONS = (".txt", ".md", ".markdown", ".html", ".htm")
+
+
+def write_document(path, *, large = False):
+ text = "Revenue increased substantially. Café operations expanded."
+ paragraphs = [text]
+ if large:
+ paragraphs += [f"Quarter {i}: revenue increased across all markets." for i in range(400)]
+ paragraphs.append("Zebramarker concludes the report.")
+ if path.suffix == ".pdf":
+ pymupdf = pytest.importorskip("pymupdf")
+ with pymupdf.open() as doc:
+ for i in range(0, len(paragraphs), 30):
+ page = doc.new_page()
+ assert (
+ page.insert_textbox(
+ pymupdf.Rect(40, 40, 550, 800),
+ "\n".join(paragraphs[i : i + 30]),
+ fontsize = 10,
+ )
+ >= 0
+ )
+ doc.save(path)
+ elif path.suffix == ".docx":
+ docx = pytest.importorskip("docx")
+ doc = docx.Document()
+ for paragraph in paragraphs[:-1]:
+ doc.add_paragraph(paragraph)
+ table = doc.add_table(rows = 1, cols = 2)
+ table.cell(0, 0).text = "Conclusion"
+ table.cell(0, 1).text = paragraphs[-1]
+ doc.save(path)
+ elif path.suffix in (".html", ".htm"):
+ path.write_text(
+ "<html><style>hiddenstylemarker</style><script>hiddenscriptmarker</script>"
+ + "".join(f"<p>{p}</p>" for p in paragraphs)
+ + "</html>",
+ encoding = "utf-8",
+ )
+ else:
+ path.write_text("\n\n".join(paragraphs), encoding = "utf-8")
+ return path
+
+
+def persist_document(monkeypatch):
+ monkeypatch.setenv(
+ leases.LEASE_SECRET_ENV, base64.urlsafe_b64encode(SECRET).decode("ascii").rstrip("=")
+ )
+ monkeypatch.setattr(leases, "_CACHED_LEASE_SECRET", None)
+
+ def persist(path, transport):
+ if transport == "desktop":
+ return _resolve_document_upload(None, _sign(path))
+ with path.open("rb") as handle:
+ return _resolve_document_upload(UploadFile(file = handle, filename = path.name), None)
+
+ return persist
+
+
[email protected]("extension", EXTENSIONS)
[email protected]("transport", ["browser", "desktop"])
+def test_formats_finish_and_remain_searchable(
+ rag_home, stub_embeddings, tmp_path, monkeypatch, persist_document, extension, transport
+):
+ assert set(EXTENSIONS) == config.UPLOAD_EXTS
+ source = write_document(tmp_path / f"report{extension}", large = True)
+ stored, filename = persist_document(source, transport)
+ assert Path(stored).read_bytes() == source.read_bytes()
+ vision_calls = []
+ monkeypatch.setattr(captioner, "vision_endpoint", lambda: ("http://local", "local"))
+ monkeypatch.setattr(captioner, "_caption_one", lambda *args: vision_calls.append("caption"))
+ monkeypatch.setattr(captioner, "_ocr_one", lambda *args: vision_calls.append("ocr"))
+ scope = store.thread_scope("formats")
+ doc_id, job = ingestion.start_ingestion(
+ scope, None, "formats", filename, stored, caption = True, ocr = True, background = False
+ )
+ events = list(ingestion.job_events(job))
+ assert events[-1]["type"] == "complete"
+ progress = [e["progress"] for e in events if e["type"] == "progress"]
+ assert progress == sorted(progress)
+ assert not vision_calls # Text-only fixtures need no vision.
+ conn = rag_db.get_connection()
+ try:
+ doc = store.get_document(conn, doc_id)
+ assert doc["status"] == "completed" and doc["num_chunks"] > 1
+ assert store.search_lexical(conn, scope, "revenue", 5)
+ assert store.search_lexical(
+ conn, scope, "zebramarker", 5
+ ) # Preserve the final paragraph or table cell.
+ assert not store.search_lexical(conn, scope, "hiddenscriptmarker", 5)
+ finally:
+ conn.close()
+ duplicate_path, filename = persist_document(source, transport)
+ duplicate_id, duplicate_job = ingestion.start_ingestion(
+ scope, None, "formats", filename, duplicate_path, background = False
+ )
+ assert duplicate_id == doc_id
+ assert ingestion.get_job_status(duplicate_job)["status"] == "completed"
+ assert Path(stored).exists() and source.exists()
+
+
[email protected]("extension", TEXT_EXTENSIONS)
+ "encoding,bom",
+ [
+ ("utf-8", codecs.BOM_UTF8),
+ ("utf-16-le", codecs.BOM_UTF16_LE),
+ ("utf-16-be", codecs.BOM_UTF16_BE),
+ ("utf-32-le", codecs.BOM_UTF32_LE),
+ ("utf-32-be", codecs.BOM_UTF32_BE),
+ ],
+)
+def test_unicode_text_upload_is_decoded_before_indexing(
+ rag_home, stub_embeddings, tmp_path, extension, encoding, bom
+):
+ text = "Café revenue increased. 東京 operations expanded. Zebramarker concludes the report."
+ raw = f"<p>{text}</p>" if extension in (".html", ".htm") else text
+ path = tmp_path / f"unicode{extension}"
+ path.write_bytes(bom + raw.encode(encoding))
+ pages = parsers.parse(str(path))
+ assert pages[0].text == text
+ scope = store.thread_scope("unicode")
+ _, job = ingestion.start_ingestion(
+ scope, None, "unicode", path.name, str(path), background = False
+ )
+ assert ingestion.get_job_status(job)["status"] == "completed"
+ conn = rag_db.get_connection()
+ try:
+ assert store.search_lexical(conn, scope, "zebramarker", 5)
+ finally:
+ conn.close()
+
+
[email protected]("extension", EXTENSIONS)
[email protected]("transport", ["browser", "desktop"])
+def test_zero_byte_formats_are_rejected_before_indexing(
+ rag_home, tmp_path, persist_document, extension, transport
+):
+ path = tmp_path / f"empty{extension}"
+ path.write_bytes(b"")
+ with pytest.raises(HTTPException) as error:
+ persist_document(path, transport)
+ assert error.value.status_code == 400
+
+
[email protected]("extension", [".pdf", ".docx"])
+def test_corrupt_documents_fail_then_valid_upload_succeeds(
+ rag_home, stub_embeddings, tmp_path, extension
+):
+ path = tmp_path / f"broken{extension}"
+ path.write_bytes(b"not a document archive")
+ scope = store.thread_scope("corrupt")
+ _, job = ingestion.start_ingestion(
+ scope, None, "corrupt", path.name, str(path), background = False
+ )
+ assert ingestion.get_job_status(job)["status"] == "failed"
+ assert list(ingestion.job_events(job))[-1]["type"] == "error"
+ write_document(path)
+ _, retry = ingestion.start_ingestion(
+ scope, None, "corrupt", path.name, str(path), background = False
+ )
+ assert ingestion.get_job_status(retry)["status"] == "completed"
diff --git a/studio/backend/tests/test_rag_upload_progress.py b/studio/backend/tests/test_rag_upload_progress.py
new file mode 100644
index 00000000000..09d0c650b39
--- /dev/null
+++ b/studio/backend/tests/test_rag_upload_progress.py
@@ -0,0 +1,432 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Uploads must follow real work and never report an in-flight duplicate as indexed."""
+
+import runpy
+import threading
+
+import pytest
+
+from core.rag import captioner, config, ingestion, parsers, store
+from storage import rag_db
+from .test_rag_upload_formats import EXTENSIONS, write_document
+
+
+def test_captioning_is_opt_in(monkeypatch):
+ monkeypatch.delenv("RAG_CAPTION_IMAGES", raising = False)
+ assert runpy.run_path(config.__file__)["CAPTION_IMAGES"] is False
+ monkeypatch.setenv("RAG_CAPTION_IMAGES", "1")
+ assert runpy.run_path(config.__file__)["CAPTION_IMAGES"] is True
+
+
[email protected]("fails", [False, True])
[email protected]("extension", EXTENSIONS)
+def test_duplicate_upload_follows_original_job(
+ rag_home, stub_embeddings, monkeypatch, fails, extension
+):
+ from utils.paths import ensure_dir, rag_uploads_root
+
+ uploads = ensure_dir(rag_uploads_root())
+ original = write_document(uploads / f"report{extension}")
+ duplicate = uploads / f"copy{extension}"
+ duplicate.write_bytes(original.read_bytes())
+ started, release = threading.Event(), threading.Event()
+ parse = parsers.parse
+
+ def blocked_parse(path):
+ started.set()
+ assert release.wait(5)
+ if fails:
+ raise ValueError("Cannot read document")
+ return parse(path)
+
+ monkeypatch.setattr(parsers, "parse", blocked_parse)
+ scope = store.thread_scope("upload-test")
+ doc_id, job_id = ingestion.start_ingestion(
+ scope, None, "upload-test", original.name, str(original)
+ )
+ try:
+ assert started.wait(5)
+ duplicate_id, duplicate_job = ingestion.start_ingestion(
+ scope, None, "upload-test", duplicate.name, str(duplicate)
+ )
+ assert duplicate_id == doc_id
+ assert duplicate_job == job_id
+ assert ingestion.get_job_status(duplicate_job)["status"] == "running"
+ assert not duplicate.exists()
+ assert original.exists()
+ finally:
+ release.set()
+ events = list(ingestion.job_events(job_id))
+ assert events[-1]["type"] == ("error" if fails else "complete")
+ assert ingestion.get_job_status(job_id)["status"] == ("failed" if fails else "completed")
+
+
+def test_same_path_dedup_keeps_original_file(rag_home, stub_embeddings):
+ from utils.paths import ensure_dir, rag_uploads_root
+
+ original = ensure_dir(rag_uploads_root()) / "report.txt"
+ original.write_text("Quarterly revenue increased substantially.")
+ scope = store.thread_scope("same-path")
+ doc_id, _ = ingestion.start_ingestion(
+ scope, None, "same-path", original.name, str(original), background = False
+ )
+ repeated_id, _ = ingestion.start_ingestion(
+ scope, None, "same-path", original.name, str(original), background = False
+ )
+ assert repeated_id == doc_id
+ assert original.exists()
+
+
+def test_vision_progress_includes_empty_responses(monkeypatch):
+ monkeypatch.setattr(captioner, "_caption_one", lambda *args: None)
+ monkeypatch.setattr(captioner, "_ocr_one", lambda *args: None)
+ figures = [parsers.ParsedImage(b"png", page, page) for page in (1, 2, 3)]
+ caption_progress, ocr_progress = [], []
+ assert (
+ captioner.caption_images(
+ figures,
+ endpoint = ("http://local", "local"),
+ on_progress = lambda done, total: caption_progress.append((done, total)),
+ )
+ == {}
+ )
+ assert (
+ captioner.ocr_pages(
+ {1: b"png", 2: b"png"},
+ endpoint = ("http://local", "local"),
+ on_progress = lambda done, total: ocr_progress.append((done, total)),
+ )
+ == {}
+ )
+ assert caption_progress == [(1, 3), (2, 3), (3, 3)]
+ assert ocr_progress == [(1, 2), (2, 2)]
+
+
+def test_indexing_reports_batches_and_keeps_text(rag_home, stub_embeddings, monkeypatch, tmp_path):
+ monkeypatch.setattr(ingestion, "_EMBED_BATCH", 1)
+ monkeypatch.setattr(config, "CHUNK_TOKENS", 4)
+ monkeypatch.setattr(config, "CHUNK_OVERLAP", 0)
+ path = tmp_path / "report.txt"
+ path.write_text("alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima")
+ scope = store.thread_scope("batch-progress")
+ doc_id, job_id = ingestion.start_ingestion(
+ scope, None, "batch-progress", path.name, str(path), background = False
+ )
+ events = list(ingestion.job_events(job_id))
+ progress = [event["progress"] for event in events if event["type"] == "progress"]
+ batches = [event for event in events if event.get("stage") == "embedding"]
+ assert len(batches) >= 4
+ assert progress == sorted(progress)
+ assert events[-1]["type"] == "complete"
+ conn = rag_db.get_connection()
+ try:
+ assert store.get_document(conn, doc_id)["num_chunks"] == 3
+ assert store.search_lexical(conn, scope, "juliet", 5)
+ finally:
+ conn.close()
+
+
+def test_figure_progress_arrives_before_render_and_after_every_tile(
+ rag_home, stub_embeddings, monkeypatch, tmp_path
+):
+ monkeypatch.setattr(
+ parsers, "parse", lambda path: [parsers.Page("Revenue increased this quarter.", 1)]
+ )
+ monkeypatch.setattr(captioner, "vision_endpoint", lambda: ("http://local", "local"))
+ monkeypatch.setattr(parsers, "pages_with_figures", lambda *args, **kwargs: [1])
+ monkeypatch.setattr(captioner, "_caption_one", lambda *args: "chart: revenue up")
+ events = []
+ emit = ingestion._emit
+
+ def record(job, event):
+ if event is not None:
+ events.append(event)
+ emit(job, event)
+
+ monkeypatch.setattr(ingestion, "_emit", record)
+
+ def render(*args, **kwargs):
+ assert events[-1]["stage"] == "captioning"
+ return [parsers.ParsedImage(b"png", 1, 1)] * 5
+
+ monkeypatch.setattr(parsers, "render_pdf_figure_tiles", render)
+ path = tmp_path / "report.pdf"
+ path.write_bytes(b"fixture")
+ from core.rag import locators
+
+ monkeypatch.setattr(locators, "pdf_regions_for_chunks", lambda *args: None)
+ _, job_id = ingestion.start_ingestion(
+ store.thread_scope("figures"),
+ None,
+ "figures",
+ path.name,
+ str(path),
+ caption = True,
+ ocr = False,
+ background = False,
+ )
+ assert ingestion.get_job_status(job_id)["status"] == "completed"
+ progress = [e["progress"] for e in events if e.get("stage") == "captioning"]
+ assert len(progress) == 6
+ assert all(a < b for a, b in zip(progress, progress[1:]))
+
+
+def test_orphaned_duplicate_is_reindexed_instead_of_reported_complete(
+ rag_home, stub_embeddings, tmp_path
+):
+ path = tmp_path / "orphan.txt"
+ path.write_text("Revenue doubled this quarter.")
+ scope = store.thread_scope("orphan")
+ conn = rag_db.get_connection()
+ try:
+ original = store.create_document(
+ conn,
+ scope = scope,
+ filename = path.name,
+ sha256 = ingestion._sha256_file(str(path)),
+ status = "pending",
+ stored_path = str(path),
+ )
+ finally:
+ conn.close()
+ replacement, job = ingestion.start_ingestion(
+ scope,
+ None,
+ "orphan",
+ path.name,
+ str(path),
+ background = False,
+ )
+ assert replacement != original
+ assert ingestion.get_job_status(job)["num_chunks"] > 0
+ conn = rag_db.get_connection()
+ try:
+ assert store.get_document(conn, original) is None
+ assert store.get_document(conn, replacement)["status"] == "completed"
+ finally:
+ conn.close()
+
+
[email protected]("status", ["pending", "running"])
+def test_failed_orphan_retry_retires_the_document_it_replaced(
+ rag_home, stub_embeddings, monkeypatch, tmp_path, status
+):
+ """A failed retry must not leave the orphan indexing: startup repair scans jobs, and
+ the orphan has none in flight, so nothing else would move it off ``pending``."""
+ path = tmp_path / "orphan.txt"
+ path.write_text("Revenue doubled this quarter.")
+ scope = store.thread_scope("orphan-failure")
+ conn = rag_db.get_connection()
+ try:
+ original = store.create_document(
+ conn,
+ scope = scope,
+ filename = path.name,
+ sha256 = ingestion._sha256_file(str(path)),
+ status = status,
+ stored_path = str(path),
+ )
+ finally:
+ conn.close()
+
+ def unreadable(*_args, **_kwargs):
+ raise ValueError("Cannot read document")
+
+ monkeypatch.setattr(parsers, "parse", unreadable)
+ replacement, job = ingestion.start_ingestion(
+ scope,
+ None,
+ "orphan-failure",
+ path.name,
+ str(path),
+ background = False,
+ )
+ assert replacement != original
+ assert ingestion.get_job_status(job)["status"] == "failed"
+ conn = rag_db.get_connection()
+ try:
+ assert store.get_document(conn, original) is None
+ assert store.get_document(conn, replacement)["status"] == "failed"
+ finally:
+ conn.close()
+
+
+def test_orphan_retry_losing_its_lease_still_retires_the_orphan(
+ rag_home, stub_embeddings, monkeypatch, tmp_path
+):
+ """A lost lease ends the retry for good: only ``_new_job`` claims an ingestion lease, so
+ nothing relaunches the work and the orphan would keep the scope indexing."""
+ path = tmp_path / "reclaimed.txt"
+ path.write_text("Revenue doubled this quarter.")
+ scope = store.thread_scope("orphan-reclaimed")
+ conn = rag_db.get_connection()
+ try:
+ original = store.create_document(
+ conn,
+ scope = scope,
+ filename = path.name,
+ sha256 = ingestion._sha256_file(str(path)),
+ status = "running",
+ stored_path = str(path),
+ )
+ finally:
+ conn.close()
+
+ def reclaimed(*_args, **_kwargs):
+ return False
+
+ monkeypatch.setattr(ingestion.job_leases, "renew_owned", reclaimed)
+ replacement, _ = ingestion.start_ingestion(
+ scope,
+ None,
+ "orphan-reclaimed",
+ path.name,
+ str(path),
+ background = False,
+ )
+ assert replacement != original
+ conn = rag_db.get_connection()
+ try:
+ assert store.get_document(conn, original) is None
+ finally:
+ conn.close()
+ # The replacement still holds a non-terminal job, so startup repair reaches it; the orphan
+ # held none, which is the whole reason it had to be retired here.
+ rag_db.reconcile_orphaned_ingestion_jobs()
+ conn = rag_db.get_connection()
+ try:
+ rows = conn.execute("SELECT status FROM documents WHERE scope=?", (scope,)).fetchall()
+ finally:
+ conn.close()
+ assert not any(row["status"] in {"pending", "running"} for row in rows)
+
+
+def test_failed_reindex_keeps_the_completed_document_it_replaced(
+ rag_home, stub_embeddings, monkeypatch, tmp_path
+):
+ """The stale-embedder retry replaces a searchable document, so a failure keeps it."""
+ path = tmp_path / "stale.txt"
+ path.write_text("Revenue doubled this quarter.")
+ scope = store.thread_scope("stale-embedder")
+ original, _ = ingestion.start_ingestion(
+ scope, None, "stale-embedder", path.name, str(path), background = False
+ )
+ conn = rag_db.get_connection()
+ try:
+ conn.execute(
+ "UPDATE documents SET embedding_model='other-embedder' WHERE id=?", (original,)
+ )
+ conn.commit()
+ finally:
+ conn.close()
+
+ def unreadable(*_args, **_kwargs):
+ raise ValueError("Cannot read document")
+
+ monkeypatch.setattr(parsers, "parse", unreadable)
+ replacement, job = ingestion.start_ingestion(
+ scope, None, "stale-embedder", path.name, str(path), background = False
+ )
+ assert replacement != original
+ assert ingestion.get_job_status(job)["status"] == "failed"
+ conn = rag_db.get_connection()
+ try:
+ assert store.get_document(conn, original)["status"] == "completed"
+ finally:
+ conn.close()
+ assert path.exists()
+
+
+def test_orphan_retry_that_cannot_start_a_worker_retires_the_orphan(
+ rag_home, stub_embeddings, monkeypatch, tmp_path
+):
+ """``_run`` never runs when the worker cannot start, so its finally cannot retire."""
+ path = tmp_path / "unstartable.txt"
+ path.write_text("Revenue doubled this quarter.")
+ scope = store.thread_scope("orphan-unstartable")
+ conn = rag_db.get_connection()
+ try:
+ original = store.create_document(
+ conn,
+ scope = scope,
+ filename = path.name,
+ sha256 = ingestion._sha256_file(str(path)),
+ status = "pending",
+ stored_path = str(path),
+ )
+ finally:
+ conn.close()
+
+ def no_threads(*_args, **_kwargs):
+ raise RuntimeError("can't start new thread")
+
+ monkeypatch.setattr(ingestion.threading, "Thread", no_threads)
+ with pytest.raises(RuntimeError):
+ ingestion.start_ingestion(scope, None, "orphan-unstartable", path.name, str(path))
+ conn = rag_db.get_connection()
+ try:
+ assert store.get_document(conn, original) is None
+ rows = conn.execute("SELECT status FROM documents WHERE scope=?", (scope,)).fetchall()
+ finally:
+ conn.close()
+ assert not any(row["status"] in {"pending", "running"} for row in rows)
+
+
+def test_cancelling_an_orphan_retry_keeps_the_original_upload(rag_home, stub_embeddings):
+ """Deleting the replacement takes its upload, so the orphan holds the last copy: fail that
+ row rather than retire it, which frees the scope without discarding the file."""
+ from utils.paths import ensure_dir, rag_uploads_root
+
+ uploads = ensure_dir(rag_uploads_root())
+ original_file = uploads / "original.txt"
+ original_file.write_text("Revenue doubled this quarter.")
+ retry_file = uploads / "retry.txt"
+ retry_file.write_bytes(original_file.read_bytes())
+ scope = store.thread_scope("cancel-orphan")
+ conn = rag_db.get_connection()
+ try:
+ original = store.create_document(
+ conn,
+ scope = scope,
+ filename = original_file.name,
+ sha256 = ingestion._sha256_file(str(original_file)),
+ status = "pending",
+ stored_path = str(original_file),
+ )
+ finally:
+ conn.close()
+
+ started, release = threading.Event(), threading.Event()
+ parse = parsers.parse
+
+ def blocked_parse(path):
+ started.set()
+ assert release.wait(5)
+ return parse(path)
+
+ monkeypatch = pytest.MonkeyPatch()
+ monkeypatch.setattr(parsers, "parse", blocked_parse)
+ try:
+ replacement, job = ingestion.start_ingestion(
+ scope, None, "cancel-orphan", retry_file.name, str(retry_file)
+ )
+ assert started.wait(5)
+ conn = rag_db.get_connection()
+ try:
+ store.delete_document(conn, replacement)
+ finally:
+ conn.close()
+ ingestion._remove_upload(str(retry_file))
+ finally:
+ release.set()
+ list(ingestion.job_events(job))
+ monkeypatch.undo()
+
+ assert original_file.exists()
+ conn = rag_db.get_connection()
+ try:
+ assert store.get_document(conn, original)["status"] == "failed"
+ finally:
+ conn.close()
diff --git a/studio/backend/tests/test_rag_upload_simulations.py b/studio/backend/tests/test_rag_upload_simulations.py
new file mode 100644
index 00000000000..156582f6e62
--- /dev/null
+++ b/studio/backend/tests/test_rag_upload_simulations.py
@@ -0,0 +1,121 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Upload races and filesystem aliases on real temporary files."""
+
+import os
+import threading
+import unicodedata
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import pytest
+
+from core.rag import ingestion, parsers, store
+from storage import rag_db
+from utils.paths import ensure_dir, rag_uploads_root
+from .test_rag_upload_formats import EXTENSIONS, write_document
+
+
[email protected]("extension", EXTENSIONS)
[email protected]("fails", [False, True])
+def test_concurrent_duplicates_share_one_worker(
+ rag_home, stub_embeddings, monkeypatch, extension, fails
+):
+ uploads = ensure_dir(rag_uploads_root())
+ source = write_document(uploads / f"source{extension}")
+ copies = [source]
+ for index in range(7):
+ path = uploads / f"copy-{index}{extension}"
+ path.write_bytes(source.read_bytes())
+ copies.append(path)
+ release, started = threading.Event(), threading.Event()
+ calls = []
+ parse = parsers.parse
+
+ def blocked(path):
+ calls.append(path)
+ started.set()
+ assert release.wait(15)
+ if fails:
+ raise ValueError("simulated parser failure")
+ return parse(path)
+
+ monkeypatch.setattr(parsers, "parse", blocked)
+ scope = store.thread_scope("concurrent")
+
+ def upload(path):
+ return ingestion.start_ingestion(scope, None, "concurrent", path.name, str(path))
+
+ try:
+ with ThreadPoolExecutor(max_workers = 8) as pool:
+ results = list(pool.map(upload, copies))
+ assert len(set(results)) == 1
+ assert started.wait(10)
+ assert len(calls) == 1
+ assert Path(calls[0]).exists()
+ finally:
+ release.set()
+ document, job = results[0]
+ events = list(ingestion.job_events(job))
+ assert events[-1]["type"] == ("error" if fails else "complete")
+ conn = rag_db.get_connection()
+ try:
+ assert len(store.list_documents(conn, scope)) == 1
+ assert store.get_document(conn, document)["status"] == ("failed" if fails else "completed")
+ if not fails:
+ assert store.search_lexical(conn, scope, "zebramarker", 5)
+ finally:
+ conn.close()
+
+
+def test_case_alias_cannot_delete_the_original_upload(rag_home, stub_embeddings):
+ original = ensure_dir(rag_uploads_root()) / "report.txt"
+ original.write_text("Revenue increased. Zebramarker concludes the report.", encoding = "utf-8")
+ alias = original.with_name("REPORT.TXT")
+ if not alias.exists() or not os.path.samefile(alias, original):
+ pytest.skip("Filesystem is case-sensitive")
+ scope = store.thread_scope("case-alias")
+ document, job = ingestion.start_ingestion(
+ scope, None, "case-alias", original.name, str(original), background = False
+ )
+ assert ingestion.get_job_status(job)["status"] == "completed"
+ duplicate, _ = ingestion.start_ingestion(
+ scope, None, "case-alias", alias.name, str(alias), background = False
+ )
+ assert duplicate == document
+ assert original.exists()
+
+
+def test_distinct_case_sensitive_upload_can_be_cleaned_up(rag_home):
+ original = ensure_dir(rag_uploads_root()) / "report.txt"
+ original.write_text("keep", encoding = "utf-8")
+ duplicate = original.with_name("REPORT.TXT")
+ if duplicate.exists():
+ pytest.skip("Filesystem is case-insensitive")
+ duplicate.write_text("remove", encoding = "utf-8")
+ ingestion._remove_upload(str(duplicate), keep_path = str(original))
+ assert original.read_text(encoding = "utf-8") == "keep"
+ assert not duplicate.exists()
+
+
+def test_unicode_filename_alias_keeps_the_original(rag_home):
+ original = ensure_dir(rag_uploads_root()) / "café.txt"
+ original.write_text("keep", encoding = "utf-8")
+ alias = original.with_name(unicodedata.normalize("NFD", original.name))
+ if not alias.exists() or not os.path.samefile(alias, original):
+ pytest.skip("Filesystem distinguishes Unicode normalization forms")
+ ingestion._remove_upload(str(alias), keep_path = str(original))
+ assert original.read_text(encoding = "utf-8") == "keep"
+
+
+def test_symlink_alias_keeps_the_original(rag_home):
+ original = ensure_dir(rag_uploads_root()) / "original.txt"
+ original.write_text("keep", encoding = "utf-8")
+ alias = original.with_name("alias.txt")
+ try:
+ alias.symlink_to(original)
+ except OSError:
+ pytest.skip("Creating symlinks requires additional privileges")
+ ingestion._remove_upload(str(alias), keep_path = str(original))
+ assert original.read_text(encoding = "utf-8") == "keep"
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index 6928489d9a2..7f2bc7e6eee 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -187,7 +187,7 @@ export const DEFAULT_RAG_AUTOINJECT_MIN_SCORE = 0.7;
// OCR scanned/image-only PDF pages at ingest; off skips the extra vision pass (only matters with a vision chat model).
export const DEFAULT_RAG_OCR = true;
// Describe figures/charts in PDFs at ingest so they become searchable; no-op without a vision model.
-export const DEFAULT_RAG_CAPTION = true;
+export const DEFAULT_RAG_CAPTION = false;
export const DEFAULT_RESEARCH_WEBSITE_POLICY: ResearchWebsitePolicy = {
allowedDomains: [],
blockedDomains: [],
diff --git a/studio/frontend/src/features/rag/api/rag-api.ts b/studio/frontend/src/features/rag/api/rag-api.ts
index 18294e90f06..672a4911a22 100644
--- a/studio/frontend/src/features/rag/api/rag-api.ts
+++ b/studio/frontend/src/features/rag/api/rag-api.ts
@@ -47,12 +47,13 @@ export function isRagClientError(error: unknown): boolean {
async function ragRequest<T>(
path: string,
- init?: { method?: string; body?: object },
+ init?: { method?: string; body?: object; signal?: AbortSignal },
): Promise<T> {
const response = await authFetch(`${RAG_BASE}${path}`, {
method: init?.method,
headers: init?.body ? { "Content-Type": "application/json" } : undefined,
body: init?.body ? JSON.stringify(init.body) : undefined,
+ signal: init?.signal,
});
if (response.status === 204) {
noteRagResponse(204, null);
@@ -720,8 +721,8 @@ export async function deleteDocument(
return result;
}
-export function getJob(jobId: string): Promise<IndexJob> {
- return ragRequest(`/jobs/${encodeURIComponent(jobId)}`);
+export function getJob(jobId: string, signal?: AbortSignal): Promise<IndexJob> {
+ return ragRequest(`/jobs/${encodeURIComponent(jobId)}`, { signal });
}
/** Longest gap between frames before a stream is treated as buffered by a proxy. */
@@ -742,28 +743,50 @@ async function openEventStream(
return response.body;
}
+// One budget for every RAG stream: HTTP/1.1 allows six connections per origin, so counting
+// only document jobs let folder syncs fill the pool and stall the upload POSTs anyway.
+// Both callers already poll when a stream throws.
+const MAX_RAG_STREAMS = 4;
+let activeRagStreams = 0;
+
+async function* boundedEventStream<T>(
+ path: string,
+ signal?: AbortSignal,
+ stallMs?: number,
+): AsyncGenerator<T> {
+ if (activeRagStreams >= MAX_RAG_STREAMS) {
+ throw new Error("RAG stream capacity reached");
+ }
+ activeRagStreams += 1;
+ try {
+ const body = await openEventStream(`${RAG_BASE}${path}`, signal);
+ yield* readSseJsonEvents<T>(body, stallMs);
+ } finally {
+ activeRagStreams -= 1;
+ }
+}
+
// sse; returns on [DONE]. transport errors propagate so callers can poll getJob
-export async function* streamJobEvents(
+export function streamJobEvents(
jobId: string,
signal?: AbortSignal,
): AsyncGenerator<JobEvent> {
// no stall bound: this consumer reads an early end as a finished job
- const body = await openEventStream(
- `${RAG_BASE}/jobs/${encodeURIComponent(jobId)}/events`,
+ return boundedEventStream<JobEvent>(
+ `/jobs/${encodeURIComponent(jobId)}/events`,
signal,
);
- yield* readSseJsonEvents<JobEvent>(body);
}
-export async function* streamFolderSyncJobEvents(
+export function streamFolderSyncJobEvents(
jobId: string,
signal?: AbortSignal,
): AsyncGenerator<FolderSyncJobEvent> {
- const body = await openEventStream(
- `${RAG_BASE}/linked-folder-jobs/${encodeURIComponent(jobId)}/events`,
+ return boundedEventStream<FolderSyncJobEvent>(
+ `/linked-folder-jobs/${encodeURIComponent(jobId)}/events`,
signal,
+ SSE_STALL_MS,
);
- yield* readSseJsonEvents<FolderSyncJobEvent>(body, SSE_STALL_MS);
}
export function getPreviewTarget(
diff --git a/studio/frontend/src/features/rag/components/document-status-chip.tsx b/studio/frontend/src/features/rag/components/document-status-chip.tsx
index 49fa4fcbab4..ca6907e9b3a 100644
--- a/studio/frontend/src/features/rag/components/document-status-chip.tsx
+++ b/studio/frontend/src/features/rag/components/document-status-chip.tsx
@@ -9,10 +9,20 @@ import { HugeiconsIcon } from "@hugeicons/react";
import { XIcon } from "lucide-react";
import type { DocumentStatus } from "../types/rag";
+const STAGE_LABELS: Record<string, string> = {
+ parsing: "Reading document",
+ ocr: "Reading scanned pages",
+ captioning: "Reading charts and figures",
+ chunking: "Preparing text",
+ embedding: "Indexing text",
+ storing: "Saving document",
+};
+
export function DocumentStatusChip({
filename,
status,
progress,
+ stage,
error,
onRemove,
shared = false,
@@ -20,6 +30,7 @@ export function DocumentStatusChip({
filename: string;
status: DocumentStatus;
progress?: number | null;
+ stage?: string | null;
error?: string | null;
onRemove?: () => void;
/** Indexed for the whole project rather than this one chat: swap the file
@@ -33,9 +44,11 @@ export function DocumentStatusChip({
size="sm"
title={
error ??
- (shared
- ? `${filename} — shared with every chat in this project`
- : filename)
+ (processing && stage && STAGE_LABELS[stage]
+ ? `${filename} — ${STAGE_LABELS[stage]}`
+ : shared
+ ? `${filename} — shared with every chat in this project`
+ : filename)
}
className={cn(
"rounded-full inline-flex items-center gap-1.5 max-w-[16rem]",
diff --git a/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx b/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx
index 6be88320352..761767ebe78 100644
--- a/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx
+++ b/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx
@@ -411,6 +411,7 @@ function KnowledgeBaseDocuments({
filename={doc.filename}
status={doc.status}
progress={doc.progress}
+ stage={doc.stage}
error={doc.error}
onRemove={
doc.id.startsWith("pending_") || isLinkedFolderManaged(doc)
diff --git a/studio/frontend/src/features/rag/components/project-sources-panel.tsx b/studio/frontend/src/features/rag/components/project-sources-panel.tsx
index 25e645f1639..02ed603f909 100644
--- a/studio/frontend/src/features/rag/components/project-sources-panel.tsx
+++ b/studio/frontend/src/features/rag/components/project-sources-panel.tsx
@@ -196,6 +196,7 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) {
filename={doc.filename}
status={doc.status}
progress={doc.progress}
+ stage={doc.stage}
error={doc.error}
onRemove={
doc.id.startsWith("pending_") || isLinkedFolderManaged(doc)
diff --git a/studio/frontend/src/features/rag/components/retrieval-settings-section.tsx b/studio/frontend/src/features/rag/components/retrieval-settings-section.tsx
index 9c91360d44e..9cd7199af40 100644
--- a/studio/frontend/src/features/rag/components/retrieval-settings-section.tsx
+++ b/studio/frontend/src/features/rag/components/retrieval-settings-section.tsx
@@ -244,7 +244,7 @@ export function RetrievalSettingsSection() {
</InfoHint>
</span>
<span className="text-ui-12 leading-[1.3] text-muted-foreground">
- Read charts and diagrams when attaching.
+ Optional vision pass; can take several minutes.
</span>
</div>
<Switch
diff --git a/studio/frontend/src/features/rag/components/thread-documents-bar.tsx b/studio/frontend/src/features/rag/components/thread-documents-bar.tsx
index 9654754d67b..984bb40bf5f 100644
--- a/studio/frontend/src/features/rag/components/thread-documents-bar.tsx
+++ b/studio/frontend/src/features/rag/components/thread-documents-bar.tsx
@@ -355,7 +355,7 @@ export function ThreadDocumentsBar({
// (ProjectLanding's pendingNewThreadId branch) and drop the just-attached chips.
const [materializedId, setMaterializedId] = useState<string | null>(null);
const effectiveThreadId = threadId ?? materializedId;
- const initPromiseRef = useRef<Promise<string | null> | null>(null);
+ const initPromiseRef = useRef<Promise<string> | null>(null);
const initGenerationRef = useRef(0);
const hadThreadIdRef = useRef(threadId !== null);
useEffect(() => {
@@ -461,14 +461,10 @@ export function ThreadDocumentsBar({
// Materialize the thread id on first use; ref-deduped so a double-click can't
// start two threads. A thread switch gets separate work even if the prior request is pending.
- const ensureThreadId = useCallback((): Promise<string | null> => {
+ const ensureThreadId = useCallback((): Promise<string> => {
if (effectiveThreadId) {
return requireStoredThread(effectiveThreadId).then(
() => effectiveThreadId,
- () => {
- toast.error("Couldn't start a chat for these documents");
- return null;
- },
);
}
const current = initPromiseRef.current;
@@ -497,10 +493,6 @@ export function ThreadDocumentsBar({
setMaterializedId(remoteId);
}
return remoteId;
- })
- .catch(() => {
- toast.error("Couldn't start a chat for these documents");
- return null;
});
initPromiseRef.current = pending;
const clear = () => {
@@ -526,12 +518,11 @@ export function ThreadDocumentsBar({
);
return;
}
- // The id as a promise, so upload() flips its in-flight guard before
- // materialization re-renders us: on the first click `scope` is null.
- const threadScope = ensureThreadId().then((id) =>
- id ? ({ type: "thread", threadId: id } as const) : null,
- );
- void upload(items, threadScope);
+ // Filter duplicates before initializing the chat.
+ void upload(items, async () => ({
+ type: "thread",
+ threadId: await ensureThreadId(),
+ }));
},
[ensureThreadId, projectId, sharesWithProject, upload, uploadToProject],
);
@@ -677,6 +668,7 @@ export function ThreadDocumentsBar({
filename={doc.filename}
status={doc.status}
progress={doc.progress}
+ stage={doc.stage}
error={doc.error}
shared={true}
onRemove={
@@ -692,6 +684,7 @@ export function ThreadDocumentsBar({
filename={doc.filename}
status={doc.status}
progress={doc.progress}
+ stage={doc.stage}
error={doc.error}
onRemove={
doc.id.startsWith("pending_")
diff --git a/studio/frontend/src/features/rag/components/use-rag-documents.ts b/studio/frontend/src/features/rag/components/use-rag-documents.ts
index bb96fde8974..e2c10cbb7a4 100644
--- a/studio/frontend/src/features/rag/components/use-rag-documents.ts
+++ b/studio/frontend/src/features/rag/components/use-rag-documents.ts
@@ -31,6 +31,7 @@ import { resolveVisionOverrides } from "./vision-overrides";
export interface TrackedDocument extends RagDocument {
progress?: number | null;
+ stage?: string | null;
}
/** Matches the backend's folder scan interval, so a job it starts is counted
@@ -120,6 +121,18 @@ export function useRagDocuments(
// True while upload() runs, so the scope-change effect can tell a real switch
// from lazy thread materialization mid-upload (which must not reset).
const uploadInFlightRef = useRef(false);
+ const uploadGenerationRef = useRef(0);
+ const activeUploadsRef = useRef(new Set<object>());
+ useEffect(
+ () => () => {
+ uploadGenerationRef.current += 1;
+ activeUploadsRef.current.clear();
+ uploadInFlightRef.current = false;
+ for (const controller of trackedJobs.current.values()) controller.abort();
+ trackedJobs.current.clear();
+ },
+ [],
+ );
// Refreshes fire from several places at once (a mutation invalidates before
// and after, the poll ticks, the scope changes) and can complete out of order,
// so only the newest publishes: an earlier one landing last would restore the
@@ -155,12 +168,20 @@ export function useRagDocuments(
if (trackedJobs.current.has(jobId)) return;
const controller = new AbortController();
trackedJobs.current.set(jobId, controller);
+ const generation = uploadGenerationRef.current;
+ const stale = () =>
+ controller.signal.aborted || generation !== uploadGenerationRef.current;
+ const forget = () => {
+ if (trackedJobs.current.get(jobId) === controller)
+ trackedJobs.current.delete(jobId);
+ };
const finish = (
status: TerminalJobStatus,
error?: string | null,
numChunks?: number | null,
) => {
+ if (stale()) return forget();
if (status === "cancelled") {
sigByDocId.current.delete(documentId);
setDocuments((rows) => rows.filter((row) => row.id !== documentId));
@@ -181,16 +202,20 @@ export function useRagDocuments(
...(numChunks != null ? { numChunks } : {}),
});
}
- trackedJobs.current.delete(jobId);
+ forget();
};
(async () => {
+ // Past the shared budget in rag-api, streamJobEvents throws and this falls
+ // through to the poll below.
try {
for await (const ev of streamJobEvents(jobId, controller.signal)) {
+ if (stale()) return forget();
if (ev.type === "progress") {
patchDoc(documentId, {
status: "running",
progress: ev.progress ?? null,
+ stage: ev.stage ?? null,
});
} else if (ev.type === "complete") {
finish("completed", null, ev.num_chunks);
@@ -204,38 +229,45 @@ export function useRagDocuments(
}
}
// Stream ended with no terminal frame: reconcile.
- const job = await getJob(jobId);
+ if (stale()) return forget();
+ const job = await getJob(jobId, controller.signal);
const terminal = terminalJobStatus(job.status);
- finish(terminal ?? "completed", job.error, job.numChunks);
+ if (terminal) {
+ finish(terminal, job.error, job.numChunks);
+ return;
+ }
} catch {
- if (controller.signal.aborted) {
- trackedJobs.current.delete(jobId);
+ if (stale()) {
+ forget();
return;
}
- // SSE unavailable: poll to a terminal state.
- try {
- for (let i = 0; i < 600; i++) {
- if (controller.signal.aborted) break;
- const job = await getJob(jobId);
- const terminal = terminalJobStatus(job.status);
- if (terminal) {
- return finish(
- terminal,
- terminal === "failed"
- ? (job.error ?? "Indexing failed")
- : job.error,
- job.numChunks,
- );
- }
- patchDoc(documentId, {
- status: job.status === "running" ? "running" : "pending",
- progress: job.progress ?? null,
- });
- await new Promise((r) => setTimeout(r, 1500));
+ }
+ // Poll until the persisted job reaches a terminal state.
+ try {
+ while (!stale()) {
+ const job = await getJob(jobId, controller.signal);
+ if (stale()) return forget();
+ const terminal = terminalJobStatus(job.status);
+ if (terminal) {
+ return finish(
+ terminal,
+ terminal === "failed"
+ ? (job.error ?? "Indexing failed")
+ : job.error,
+ job.numChunks,
+ );
}
- } catch {
- trackedJobs.current.delete(jobId);
+ patchDoc(documentId, {
+ status: job.status === "running" ? "running" : "pending",
+ progress: job.progress ?? null,
+ stage: job.stage ?? null,
+ });
+ await new Promise((r) => setTimeout(r, 1500));
}
+ } catch {
+ // A list refresh can restart tracking after a request fails.
+ } finally {
+ forget();
}
})();
},
@@ -262,7 +294,7 @@ export function useRagDocuments(
return tracked &&
tracked.progress != null &&
row.status !== "completed"
- ? { ...row, progress: tracked.progress }
+ ? { ...row, progress: tracked.progress, stage: tracked.stage }
: row;
});
// Keep optimistic chips (not yet listed) so a refresh racing an upload
@@ -343,6 +375,10 @@ export function useRagDocuments(
// null scope starts no replacement request to outrank it, so without this
// its response would repopulate the list that is about to be cleared.
refreshSeq.current += 1;
+ uploadGenerationRef.current += 1;
+ activeUploadsRef.current.clear();
+ uploadInFlightRef.current = false;
+ setUploading(false);
// Scope changes intentionally clear the old scope before fetching the new
// one. Keep this synchronous so React StrictMode's setup/cleanup replay
// cannot cancel the only refresh after prevScopeKeyRef has advanced.
@@ -451,18 +487,18 @@ export function useRagDocuments(
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [projectScopeId]);
- // POST one file, then swap its optimistic chip (`tempId`) to the real id; drop
- // the chip if the backend deduped. `seenIds` holds ids present/added this batch.
+ // Replace pending chips, deduplicating concurrent upload responses.
const uploadOne = useCallback(
async (
item: RagUploadItem,
- seenIds: Set<string>,
activeScope: RagDocumentScope,
tempId: string,
+ generation: number,
) => {
const name = itemName(item);
try {
const { ocr, caption } = await resolveVisionOverrides();
+ if (generation !== uploadGenerationRef.current) return;
// Leases are short-lived, so mint one per upload rather than at drop time.
const file =
item.kind === "file"
@@ -472,6 +508,7 @@ export function useRagDocuments(
await consumeNativePathToken(item.token, "attach")
).nativePathLease,
};
+ if (generation !== uploadGenerationRef.current) return;
const result =
activeScope.type === "kb"
? await uploadKnowledgeBaseDocument(
@@ -493,29 +530,25 @@ export function useRagDocuments(
ocr,
caption,
);
+ if (generation !== uploadGenerationRef.current) return;
sigByDocId.current.set(result.documentId, itemSignature(item));
- if (seenIds.has(result.documentId)) {
- setDocuments((rows) => rows.filter((row) => row.id !== tempId));
- toast.info(
- `${result.filename || name} is already indexed - skipping`,
- );
- return;
- }
- seenIds.add(result.documentId);
setDocuments((rows) =>
- rows.map((row) =>
- row.id === tempId
- ? {
- ...row,
- id: result.documentId,
- filename: result.filename || row.filename,
- status: "running",
- }
- : row,
- ),
+ rows.some((row) => row.id === result.documentId)
+ ? rows.filter((row) => row.id !== tempId)
+ : rows.map((row) =>
+ row.id === tempId
+ ? {
+ ...row,
+ id: result.documentId,
+ filename: result.filename || row.filename,
+ status: "running",
+ }
+ : row,
+ ),
);
trackJob(result.jobId, result.documentId, result.filename || name);
} catch (err) {
+ if (generation !== uploadGenerationRef.current) return;
const message = err instanceof Error ? err.message : String(err);
// Drop the chip rather than show "Failed"; warn via toast.
setDocuments((rows) => rows.filter((row) => row.id !== tempId));
@@ -531,8 +564,14 @@ export function useRagDocuments(
const upload = useCallback(
async (
files: FileList | File[] | RagUploadItem[],
- overrideScope?: RagDocumentScope | Promise<RagDocumentScope | null>,
+ overrideScope?:
+ | RagDocumentScope
+ | Promise<RagDocumentScope | null>
+ | (() => Promise<RagDocumentScope | null>),
) => {
+ const generation = uploadGenerationRef.current;
+ const uploadToken = {};
+ activeUploadsRef.current.add(uploadToken);
// Flip the in-flight guard synchronously, before awaiting a thread id that
// may still be materializing, so the scope-change effect reads it and leaves
// job tracking and optimistic chips alone.
@@ -543,7 +582,9 @@ export function useRagDocuments(
// The composer passes its project scope explicitly, since the hook's own
// can still be null on the render that starts the upload.
const knownScope =
- overrideScope instanceof Promise ? null : (overrideScope ?? scope);
+ overrideScope instanceof Promise || typeof overrideScope === "function"
+ ? null
+ : (overrideScope ?? scope);
const uploadingProjectId =
knownScope?.type === "project" ? knownScope.projectId : null;
if (uploadingProjectId) {
@@ -561,7 +602,6 @@ export function useRagDocuments(
const item: RagUploadItem =
entry instanceof File ? { kind: "file", file: entry } : entry;
if (sigBlocksReupload(itemSignature(item))) {
- toast.info(`${itemName(item)} is already indexed - skipping`);
continue;
}
fresh.push({
@@ -581,32 +621,55 @@ export function useRagDocuments(
})),
]);
- const resolved =
- overrideScope instanceof Promise
- ? await overrideScope
- : overrideScope;
- const activeScope = resolved ?? scope;
- if (!activeScope) {
+ let activeScope: RagDocumentScope | null;
+ try {
+ activeScope =
+ overrideScope === undefined
+ ? scope
+ : typeof overrideScope === "function"
+ ? await overrideScope()
+ : await overrideScope;
+ if (!activeScope) {
+ throw new Error("Could not start a chat to attach them to.");
+ }
+ } catch (err) {
+ if (generation !== uploadGenerationRef.current) return;
// Materialization failed: drop the chips so they don't hang "pending".
const tempIds = new Set(fresh.map((f) => f.tempId));
setDocuments((rows) => rows.filter((row) => !tempIds.has(row.id)));
toast.error("Couldn't attach documents", {
- description: "Could not start a chat to attach them to.",
+ description: err instanceof Error ? err.message : String(err),
});
return;
}
- const seenIds = new Set(
- documentsRef.current
- .filter((d) => !d.id.startsWith("pending_"))
- .map((d) => d.id),
- );
+ // A batch begun with no scope is exempt from the generation bump, so materializing
+ // a thread cannot abort it. That exemption also covers navigating away mid-flight,
+ // so compare destinations here: another chat on screen means this one was left. A
+ // null key is still materializing, not a navigation.
+ const resolvedKey =
+ activeScope.type === "kb"
+ ? `kb:${activeScope.kbId}`
+ : activeScope.type === "project"
+ ? `project:${activeScope.projectId}`
+ : `thread:${activeScope.threadId}`;
+ const liveKey = liveScopeKeyRef.current;
+ if (knownScope === null && liveKey !== null && liveKey !== resolvedKey) {
+ const tempIds = new Set(fresh.map((f) => f.tempId));
+ setDocuments((rows) => rows.filter((row) => !tempIds.has(row.id)));
+ return;
+ }
+
for (const { tempId, item } of fresh) {
- await uploadOne(item, seenIds, activeScope, tempId);
+ if (generation !== uploadGenerationRef.current) return;
+ await uploadOne(item, activeScope, tempId, generation);
}
} finally {
- setUploading(false);
- uploadInFlightRef.current = false;
+ activeUploadsRef.current.delete(uploadToken);
+ if (generation === uploadGenerationRef.current) {
+ uploadInFlightRef.current = activeUploadsRef.current.size > 0;
+ setUploading(uploadInFlightRef.current);
+ }
if (uploadingProjectId) {
noteProjectWork(uploadingProjectId, -1);
}
diff --git a/studio/frontend/tests/rag-availability-marker.test.ts b/studio/frontend/tests/rag-availability-marker.test.ts
index 827e600b601..6d8172be8a0 100644
--- a/studio/frontend/tests/rag-availability-marker.test.ts
+++ b/studio/frontend/tests/rag-availability-marker.test.ts
@@ -235,11 +235,16 @@ test("every RAG response path reports availability", async () => {
/noteRagResponse\(response\.status, body\)/,
"the SSE endpoints bypass ragRequest too",
);
+ assert.match(
+ functionBody(src, "boundedEventStream"),
+ /openEventStream\(/,
+ "the budgeted stream opens without the availability check",
+ );
for (const generator of ["streamJobEvents", "streamFolderSyncJobEvents"]) {
assert.match(
functionBody(src, generator),
- /openEventStream\(/,
- `${generator} opens its stream without the availability check`,
+ /boundedEventStream</,
+ `${generator} opens its stream outside the budgeted opener`,
);
}
});
diff --git a/studio/frontend/tests/rag-upload-lifecycle.test.ts b/studio/frontend/tests/rag-upload-lifecycle.test.ts
new file mode 100644
index 00000000000..1f136394f93
--- /dev/null
+++ b/studio/frontend/tests/rag-upload-lifecycle.test.ts
@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+import { loadWithStubs } from "./helpers/module-stubs.ts";
+import { terminalJobStatus } from "../src/features/rag/types/rag.ts";
+import type {
+ IndexJob,
+ JobEvent,
+ RagDocument,
+} from "../src/features/rag/types/rag.ts";
+import type { RagDocumentScope } from "../src/features/rag/components/use-rag-documents.ts";
+
+type ScopeOverride =
+ | RagDocumentScope
+ | Promise<RagDocumentScope | null>
+ | (() => Promise<RagDocumentScope | null>);
+type Hook = {
+ documents: RagDocument[];
+ uploading: boolean;
+ hasIndexing: boolean;
+ upload: (files: File[], scope?: ScopeOverride) => Promise<void>;
+};
+
+function deferred<T>() {
+ let resolve!: (value: T) => void;
+ const promise = new Promise<T>((done) => {
+ resolve = done;
+ });
+ return { promise, resolve };
+}
+const flush = () => new Promise<void>((resolve) => setImmediate(resolve));
+
+/** Drive renders of the shipped hook with controlled network responses. */
+function harness(
+ options: {
+ filename?: string;
+ events?: () => AsyncGenerator<JobEvent>;
+ getJob?: () => Promise<IndexJob>;
+ } = {},
+) {
+ const slots: unknown[] = [];
+ const effects: Array<() => void> = [];
+ let cursor = 0;
+ const errors: string[] = [];
+ const infos: string[] = [];
+ const uploads: string[] = [];
+ const uploaded = {
+ documentId: "doc",
+ jobId: "job",
+ filename: options.filename ?? "report.pdf",
+ };
+ const lister = async () => [];
+ let scope: RagDocumentScope | null = { type: "thread", threadId: "thread" };
+ const react = {
+ useRef(value: unknown) {
+ const index = cursor++;
+ slots[index] ??= { current: value };
+ return slots[index];
+ },
+ useState(value: unknown) {
+ const index = cursor++;
+ if (!(index in slots)) slots[index] = value;
+ return [
+ slots[index],
+ (next: unknown) => {
+ slots[index] = typeof next === "function" ? next(slots[index]) : next;
+ },
+ ];
+ },
+ useCallback(fn: unknown) {
+ return fn;
+ },
+ useEffect(effect: () => void | (() => void), deps: unknown[]) {
+ const index = cursor++;
+ const previous = slots[index] as
+ { deps: unknown[]; cleanup?: () => void } | undefined;
+ if (previous && deps.every((dep, i) => Object.is(dep, previous.deps[i])))
+ return;
+ effects.push(() => {
+ previous?.cleanup?.();
+ slots[index] = { deps, cleanup: effect() };
+ });
+ },
+ };
+ const { useRagDocuments: runHook } = loadWithStubs<{
+ useRagDocuments: (
+ scope: RagDocumentScope | null,
+ lister: () => Promise<RagDocument[]>,
+ ) => Hook;
+ }>(
+ new URL(
+ "../src/features/rag/components/use-rag-documents.ts",
+ import.meta.url,
+ ),
+ {
+ react,
+ "@/features/native-intents": {},
+ "@/lib/toast": {
+ toast: {
+ error: (message: string) => errors.push(message),
+ info: (message: string) => infos.push(message),
+ },
+ },
+ "../types/rag": { terminalJobStatus },
+ "../api/rag-availability": {
+ useRagAvailabilityStore: {
+ getState: () => ({ isUnavailable: () => false }),
+ },
+ },
+ "./vision-overrides": { resolveVisionOverrides: async () => ({}) },
+ "../api/rag-api": {
+ uploadThreadDocument: async (threadId: string) => {
+ uploads.push(threadId);
+ return uploaded;
+ },
+ streamJobEvents:
+ options.events ??
+ async function* () {
+ yield { type: "complete", num_chunks: 10 } as JobEvent;
+ },
+ getJob:
+ options.getJob ??
+ (async () => ({ status: "completed", numChunks: 10 })),
+ },
+ },
+ );
+ return {
+ errors,
+ infos,
+ uploads,
+ setScope(next: RagDocumentScope | null) {
+ scope = next;
+ },
+ render() {
+ cursor = 0;
+ const result = runHook(scope, lister);
+ for (const effect of effects.splice(0)) effect();
+ return result;
+ },
+ dispose() {
+ for (const slot of slots) (slot as { cleanup?: () => void })?.cleanup?.();
+ },
+ };
+}
+
+function report(filename = "report.pdf") {
+ return new File(["report"], filename, { lastModified: 1 });
+}
+
+for (const extension of [
+ "pdf",
+ "txt",
+ "md",
+ "markdown",
+ "docx",
+ "html",
+ "htm",
+]) {
+ const filename = `report.${extension}`;
+ test(`reselecting an attached ${extension} skips chat creation and duplicate toasts`, async () => {
+ const app = harness({ filename });
+ try {
+ let hook = app.render();
+ await flush();
+ await hook.upload([report(filename)]);
+ await flush();
+ hook = app.render();
+ assert.equal(hook.documents[0]?.status, "completed");
+ let initialized = 0;
+ await hook.upload([report(filename)], async () => {
+ initialized += 1;
+ throw new Error("Thread was not persisted");
+ });
+ await flush();
+ assert.equal(initialized, 0);
+ assert.equal(app.uploads.length, 1);
+ assert.deepEqual(app.errors, []);
+ assert.deepEqual(app.infos, []);
+ assert.equal(app.render().documents.length, 1);
+ } finally {
+ app.dispose();
+ }
+ });
+
+ test(`chat initialization failure removes the ${extension} chip and reports once`, async () => {
+ const app = harness({ filename });
+ try {
+ const hook = app.render();
+ await flush();
+ await hook.upload([report(filename)], async () => {
+ throw new Error("Thread deleted");
+ });
+ assert.deepEqual(app.errors, ["Couldn't attach documents"]);
+ assert.deepEqual(app.uploads, []);
+ assert.deepEqual(app.render().documents, []);
+ assert.equal(app.render().uploading, false);
+ } finally {
+ app.dispose();
+ }
+ });
+}
+
+test("failed scope resolution cannot upload into the previous chat", async () => {
+ const app = harness();
+ try {
+ const hook = app.render();
+ await flush();
+ await hook.upload([report()], Promise.resolve(null));
+ assert.deepEqual(app.uploads, []);
+ assert.deepEqual(app.errors, ["Couldn't attach documents"]);
+ } finally {
+ app.dispose();
+ }
+});
+
+test("an ended event stream keeps the file indexing until the job completes", async () => {
+ const completion = deferred<IndexJob>();
+ let reads = 0;
+ const app = harness({
+ events: async function* () {
+ yield { type: "progress", progress: 0.4, stage: "captioning" };
+ },
+ getJob: async () => {
+ reads += 1;
+ return reads === 1
+ ? { id: "job", documentId: "doc", status: "running", progress: 0.4 }
+ : completion.promise;
+ },
+ });
+ try {
+ const hook = app.render();
+ await flush();
+ await hook.upload([report()]);
+ await flush();
+ assert.equal(app.render().documents[0]?.status, "running");
+ assert.equal(app.render().hasIndexing, true);
+ assert.equal(reads, 2);
+ completion.resolve({
+ id: "job",
+ documentId: "doc",
+ status: "completed",
+ numChunks: 12,
+ });
+ await flush();
+ assert.equal(app.render().documents[0]?.status, "completed");
+ assert.equal(app.render().documents[0]?.numChunks, 12);
+ assert.equal(app.render().hasIndexing, false);
+ } finally {
+ app.dispose();
+ }
+});
+
+test("an original job failure is preserved after its event stream ends", async () => {
+ let reads = 0;
+ const app = harness({
+ events: async function* () {},
+ getJob: async () => ({
+ id: "job",
+ documentId: "doc",
+ status: ++reads === 1 ? "running" : "failed",
+ error: "Invalid PDF",
+ }),
+ });
+ try {
+ const hook = app.render();
+ await flush();
+ await hook.upload([report()]);
+ await flush();
+ assert.deepEqual(app.render().documents, []);
+ assert.deepEqual(app.errors, ["Couldn't index report.pdf"]);
+ } finally {
+ app.dispose();
+ }
+});
+
+test("an upload begun without a scope stops at the chat the user left", async () => {
+ const app = harness();
+ try {
+ app.setScope(null);
+ let hook = app.render();
+ await flush();
+ const materialized = deferred<RagDocumentScope>();
+ const pending = hook.upload([report()], async () => materialized.promise);
+ await flush();
+ // The user picks an existing chat while the new one is still materializing.
+ app.setScope({ type: "thread", threadId: "other" });
+ hook = app.render();
+ await flush();
+ materialized.resolve({ type: "thread", threadId: "abandoned" });
+ await pending;
+ await flush();
+ assert.deepEqual(app.uploads, [], "posted into a chat the user had left");
+ assert.deepEqual(app.render().documents, []);
+ } finally {
+ app.dispose();
+ }
+});
diff --git a/tests/studio/rag_upload_browser_server.mjs b/tests/studio/rag_upload_browser_server.mjs
new file mode 100644
index 00000000000..921b617de0a
--- /dev/null
+++ b/tests/studio/rag_upload_browser_server.mjs
@@ -0,0 +1,251 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+const frontendPath = path.resolve(process.argv[2]);
+const root = path.resolve(process.argv[3]);
+const { createServer, normalizePath } = await import(
+ pathToFileURL(path.join(frontendPath, "node_modules/vite/dist/node/index.js"))
+);
+const frontend = normalizePath(frontendPath);
+let state;
+const reset = (mode = "normal") => {
+ for (const res of state?.streams ?? []) res.end();
+ for (const done of state?.waiting ?? []) done();
+ state = {
+ mode,
+ uploads: [],
+ jobs: {},
+ docs: [],
+ streams: [],
+ waiting: [],
+ eventRequests: [],
+ polls: 0,
+ };
+};
+reset();
+const json = (res, body, status = 200) => {
+ res.writeHead(status, { "Content-Type": "application/json" });
+ res.end(JSON.stringify(body));
+};
+const read = async (req) => {
+ const chunks = [];
+ for await (const data of req) chunks.push(data);
+ return Buffer.concat(chunks).toString("utf8");
+};
+const terminal = (status = "completed") => {
+ for (const job of Object.values(state.jobs))
+ Object.assign(job, {
+ status,
+ numChunks: 2,
+ error: status === "failed" ? "simulated indexing failure" : null,
+ });
+ for (const doc of state.docs) Object.assign(doc, { status, numChunks: 2 });
+};
+const fixture = async (req, res, next) => {
+ const url = new URL(req.url, "http://localhost").pathname;
+ if (url === "/__scenario") {
+ reset(JSON.parse(await read(req)).mode);
+ return json(res, { ok: true });
+ }
+ if (url === "/__state")
+ return json(res, {
+ ...state,
+ streams: state.streams.filter((r) => !r.destroyed).length,
+ waiting: state.waiting.length,
+ });
+ if (url === "/__release-one") {
+ state.waiting.shift()?.();
+ return json(res, { ok: true });
+ }
+ if (url === "/__release") {
+ terminal(state.mode.includes("fail") ? "failed" : "completed");
+ for (const done of state.waiting.splice(0)) done();
+ for (const stream of state.streams) stream.end();
+ return json(res, { ok: true });
+ }
+ if (url.endsWith("/documents") && url.startsWith("/api/rag/threads/")) {
+ const thread = url.split("/")[4];
+ if (req.method === "GET")
+ return json(res, {
+ documents: state.docs.filter((d) => d.threadId === thread),
+ });
+ const body = await read(req);
+ const filename = /filename="([^"]*)"/.exec(body)?.[1] ?? "report.txt";
+ state.uploads.push({
+ thread,
+ filename,
+ multipart: req.headers["content-type"],
+ });
+ if (state.mode === "reject")
+ return json(res, { detail: "simulated upload rejection" }, 413);
+ const id = state.mode.includes("server-duplicate")
+ ? "doc-1"
+ : `doc-${state.uploads.length}`;
+ const jobId = `job-${id}`;
+ if (!state.jobs[jobId]) {
+ state.docs.push({
+ id,
+ threadId: thread,
+ filename,
+ status: "running",
+ managed: false,
+ numChunks: 0,
+ });
+ state.jobs[jobId] = {
+ id: jobId,
+ documentId: id,
+ status: "running",
+ progress: 0.4,
+ stage: "captioning",
+ };
+ }
+ if (state.mode.startsWith("delayed"))
+ await new Promise((resolve) => state.waiting.push(resolve));
+ return json(res, { documentId: id, jobId, filename });
+ }
+ if (url.startsWith("/api/rag/jobs/")) {
+ const id = url.split("/")[4];
+ const job = state.jobs[id];
+ if (!job) return json(res, { detail: "missing job" }, 404);
+ if (!url.endsWith("/events")) {
+ state.polls++;
+ return json(res, job);
+ }
+ state.eventRequests.push({ id, method: req.method });
+ if (state.mode === "get-only" && req.method === "POST")
+ return json(res, {}, 405);
+ if (state.mode === "sse-unavailable") return json(res, {}, 503);
+ res.writeHead(200, {
+ "Content-Type": "text/event-stream",
+ "Cache-Control": "no-cache",
+ });
+ const payload = Buffer.from(
+ 'data: {"type":"progress","progress":0.4,"stage":"captioning","text":"東京 café"}\r\n\r\n',
+ );
+ for (let i = 0; i < payload.length; i += 7)
+ res.write(payload.subarray(i, i + 7));
+ if (state.mode === "early-end" || state.mode.startsWith("delayed"))
+ return res.end();
+ if (state.mode === "hold" || state.mode === "server-duplicate") {
+ state.streams.push(res);
+ return;
+ }
+ if (state.mode === "error") {
+ terminal("failed");
+ return res.end(
+ 'data: {"type":"error","error":"simulated indexing failure","stage":"error"}\n\n',
+ );
+ }
+ terminal();
+ res.end('data: {"type":"complete","num_chunks":2}\n\n');
+ return;
+ }
+ next();
+};
+
+const entry = `
+import React, {useCallback, useState} from 'react';
+import {createRoot} from 'react-dom/client';
+import {useRagDocuments} from './src/features/rag/components/use-rag-documents';
+import {listThreadDocuments} from './src/features/rag/api/rag-api';
+window.errors = []; window.resolveCalls = 0; window.pageErrors = [];
+window.addEventListener('error', event => window.pageErrors.push(event.message));
+window.addEventListener('unhandledrejection', event => window.pageErrors.push(String(event.reason)));
+let chosen = [];
+function Uploads({scopeId, setScope}) {
+ const lister = useCallback(() => scopeId ? listThreadDocuments(scopeId) : Promise.resolve([]), [scopeId]);
+ const api = useRagDocuments(scopeId ? {type:'thread',threadId:scopeId} : null, lister);
+ window.sim = { ...api, scopeId,
+ uploadNames(names, failScope=false) {
+ chosen=names.map(name => new File(['fixture document'], name, {lastModified:1}));
+ return api.upload(chosen, failScope ? async()=>{window.resolveCalls++;throw Error('simulated chat failure')} : undefined);
+ },
+ reselect() { return api.upload(chosen, async()=>{window.resolveCalls++;throw Error('duplicate initialized chat')}); },
+ materialize() { return api.upload([new File(['x'],'materialize.txt')], async()=>{setScope('thread'); return {type:'thread',threadId:'thread'};}); }
+ };
+ return React.createElement(React.Fragment, null,
+ React.createElement('input', {id:'files',type:'file',multiple:true,onChange:e=>void api.upload(e.target.files)}),
+ React.createElement('pre',{id:'state'},JSON.stringify({documents:api.documents,uploading:api.uploading,hasIndexing:api.hasIndexing,scopeId})));
+}
+function App() {
+ const [scopeId,setScope]=useState(new URLSearchParams(location.search).has('empty')?null:'thread');
+ const [mounted,setMounted]=useState(true);
+ return React.createElement(React.Fragment,null,
+ React.createElement('button',{id:'switch',onClick:()=>setScope('other')},'Switch'),
+ React.createElement('button',{id:'unmount',onClick:()=>setMounted(false)},'Unmount'),
+ mounted ? React.createElement(Uploads,{scopeId,setScope}) : React.createElement('div',{id:'gone'},'Unmounted'));
+}
+createRoot(document.getElementById('root')).render(React.createElement(React.StrictMode,null,React.createElement(App)));
+`;
+const stubs = {
+ "@/features/auth": "export const authFetch=(url,init)=>fetch(url,init);",
+ "@/lib/api-base": "export const apiUrl=url=>url;",
+ "@/lib/toast":
+ "export const toast={error:(message,details)=>window.errors.push({message,details}),info:message=>window.errors.push({message})};",
+ "@/features/native-intents":
+ "export const consumeNativePathToken=async()=>({nativePathLease:'fixture'});",
+};
+const server = await createServer({
+ configFile: false,
+ root: frontend,
+ cacheDir: path.join(root, "vite-cache"),
+ resolve: { alias: { "@": path.join(frontend, "src") } },
+ server: {
+ host: "127.0.0.1",
+ port: 18948,
+ strictPort: true,
+ fs: { allow: [frontend] },
+ },
+ plugins: [
+ {
+ name: "upload-simulation",
+ enforce: "pre",
+ resolveId(id) {
+ if (stubs[id]) return "\0" + id;
+ for (const name of Object.keys(stubs)) {
+ if (
+ normalizePath(id) ===
+ normalizePath(path.join(frontend, "src", name.slice(2)))
+ )
+ return "\0" + name;
+ }
+ if (
+ id.endsWith("/vision-overrides") ||
+ id.endsWith("/vision-overrides.ts") ||
+ id === "./vision-overrides"
+ )
+ return "\0vision";
+ if (id === "/__simulation_entry.jsx")
+ return path.join(frontend, "__simulation_entry.jsx");
+ },
+ load(id) {
+ if (id.startsWith("\0") && stubs[id.slice(1)])
+ return stubs[id.slice(1)];
+ if (id === "\0vision")
+ return "export const resolveVisionOverrides=async()=>({});";
+ if (id.endsWith("__simulation_entry.jsx")) return entry;
+ },
+ configureServer(server) {
+ server.middlewares.use(
+ (req, res, next) =>
+ void fixture(req, res, next).catch((e) =>
+ json(res, { error: String(e) }, 500),
+ ),
+ );
+ server.middlewares.use((req, res, next) => {
+ if (req.url === "/" || req.url.startsWith("/?")) {
+ res.setHeader("Content-Type", "text/html");
+ res.end(
+ '<html><div id="root"></div><script type="module" src="/__simulation_entry.jsx"></script></html>',
+ );
+ } else next();
+ });
+ },
+ },
+ ],
+});
+await server.listen();
+console.log("Upload simulation ready at http://127.0.0.1:18948");
diff --git a/tests/studio/rag_upload_browser_tests.py b/tests/studio/rag_upload_browser_tests.py
new file mode 100644
index 00000000000..8c659523a83
--- /dev/null
+++ b/tests/studio/rag_upload_browser_tests.py
@@ -0,0 +1,283 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Real browser runs of the shipped React hook and upload/SSE transport."""
+
+import json
+import os
+from pathlib import Path
+import sys
+import time
+import urllib.request
+
+from playwright.sync_api import sync_playwright
+
+ROOT = Path(os.environ.get("RAG_SIM_ART_DIR", "./temp/rag-upload")).resolve()
+ROOT.mkdir(parents = True, exist_ok = True)
+BASE = "http://127.0.0.1:18948"
+
+
+def request(path, body = None):
+ req = urllib.request.Request(
+ BASE + path,
+ data = json.dumps(body).encode() if body is not None else None,
+ headers = {"Content-Type": "application/json"},
+ )
+ return json.load(urllib.request.urlopen(req, timeout = 15))
+
+
+def wait_state(predicate):
+ deadline = time.monotonic() + 8
+ while time.monotonic() < deadline:
+ state = request("/__state")
+ if predicate(state):
+ return state
+ time.sleep(0.02)
+ raise AssertionError(f"Server condition was not reached: {state}")
+
+
+def run_case(browser, mode, action):
+ request("/__scenario", {"mode": mode})
+ context = browser.new_context()
+ page = context.new_page()
+ errors = []
+ page.on("pageerror", lambda err: errors.append(str(err)))
+ try:
+ page.goto(BASE)
+ page.wait_for_selector("#state")
+ action(page)
+ assert not errors, errors
+ assert not page.evaluate("window.pageErrors"), "Unexpected browser error"
+ finally:
+ request("/__release", {})
+ context.close()
+
+
+def complete(page):
+ # Three engines share a 2-core runner, so this waits on scheduling, not on the app.
+ page.wait_for_function(
+ "window.sim.documents.length>0 && window.sim.documents.every(d=>d.status==='completed') && !window.sim.uploading && !window.sim.hasIndexing",
+ timeout = 30000,
+ )
+ assert not page.evaluate("window.sim.hasIndexing || window.sim.uploading")
+ assert page.evaluate("window.errors") == []
+
+
+def ordinary(ext):
+ def action(page):
+ page.locator("#files").set_input_files(
+ {
+ "name": f"Café 東京.{ext}",
+ "mimeType": "application/octet-stream",
+ "buffer": b"fixture document",
+ }
+ )
+ complete(page)
+ data = request("/__state")
+ assert (
+ len(data["uploads"]) == 1
+ and "multipart/form-data; boundary=" in data["uploads"][0]["multipart"]
+ )
+
+ return action
+
+
+def reselect(page):
+ page.evaluate("window.sim.uploadNames(['report.pdf'])")
+ complete(page)
+ page.evaluate("window.sim.reselect()")
+ assert page.evaluate("window.resolveCalls") == 0
+ assert len(request("/__state")["uploads"]) == 1
+ complete(page)
+
+
+def early_end(page):
+ page.evaluate("window.sim.uploadNames(['report.txt'])")
+ page.wait_for_function("window.sim.documents[0]?.status==='running'")
+ assert page.evaluate("window.sim.hasIndexing")
+ request("/__release", {})
+ complete(page)
+
+
+def error(page):
+ page.evaluate("window.sim.uploadNames(['report.pdf'])")
+ page.wait_for_function(
+ "window.errors.length===1 && !window.sim.hasIndexing && !window.sim.uploading",
+ timeout = 30000,
+ )
+ assert not page.evaluate("window.sim.hasIndexing || window.sim.uploading")
+ assert page.evaluate("window.sim.documents") == []
+
+
+def failed_scope(page):
+ page.evaluate("window.sim.uploadNames(['report.pdf'],true)")
+ assert len(page.evaluate("window.errors")) == 1
+ assert page.evaluate("window.sim.documents") == []
+ assert not request("/__state")["uploads"]
+
+
+def duplicate_running(page):
+ page.evaluate("window.sim.uploadNames(['first.txt'])")
+ page.evaluate("window.sim.uploadNames(['same-content.txt'])")
+ page.wait_for_function("window.sim.documents.length===1 && !window.sim.uploading")
+ assert page.evaluate("window.sim.hasIndexing")
+ assert len(request("/__state")["eventRequests"]) == 1
+ request("/__release", {})
+ complete(page)
+
+
+def navigate(page):
+ page.evaluate("window.pending=window.sim.uploadNames(['old.txt']);void 0")
+ page.wait_for_function("window.sim.uploading")
+ wait_state(lambda s: s["waiting"] == 1)
+ page.locator("#switch").click()
+ page.wait_for_function("window.sim.scopeId==='other'")
+ request("/__release", {})
+ page.evaluate("window.pending")
+ page.wait_for_timeout(150)
+ assert not request("/__state")["eventRequests"], "Old upload began tracking after navigation"
+ assert page.evaluate("window.errors") == []
+ assert page.evaluate("window.sim.documents") == []
+
+
+def unmount(page):
+ page.evaluate("window.pending=window.sim.uploadNames(['old.txt']);void 0")
+ page.wait_for_function("window.sim.uploading")
+ wait_state(lambda s: s["waiting"] == 1)
+ page.locator("#unmount").click()
+ request("/__release", {})
+ page.evaluate("window.pending")
+ page.wait_for_timeout(150)
+ assert not request("/__state")["eventRequests"], "Unmounted component began job tracking"
+ assert page.evaluate("window.errors") == []
+
+
+def large_batch(page):
+ page.evaluate(
+ "window.pending=window.sim.uploadNames(Array.from({length:12},(_,i)=>'report-'+i+'.txt'));void 0"
+ )
+ deadline = time.monotonic() + 4
+ while time.monotonic() < deadline and len(request("/__state")["uploads"]) < 12:
+ time.sleep(0.05)
+ data = request("/__state")
+ assert len(data["uploads"]) == 12, f"Only {len(data['uploads'])}/12 POSTs arrived with SSE open"
+ request("/__release", {})
+ page.evaluate("window.pending")
+ complete(page)
+
+
+def overlapping_uploads(page):
+ page.evaluate("window.first=window.sim.uploadNames(['first.txt']);void 0")
+ wait_state(lambda s: s["waiting"] == 1)
+ page.evaluate("window.second=window.sim.uploadNames(['second.txt']);void 0")
+ wait_state(lambda s: s["waiting"] == 2)
+ request("/__release-one", {})
+ page.evaluate("window.first")
+ assert page.evaluate("window.sim.uploading"), (
+ "The first upload released the second upload's guard"
+ )
+ request("/__release", {})
+ page.evaluate("window.second")
+ complete(page)
+
+
+def concurrent_same_content(page):
+ page.evaluate(
+ "window.first=window.sim.uploadNames(['first.txt']);window.second=window.sim.uploadNames(['alias.txt']);void 0"
+ )
+ wait_state(lambda s: s["waiting"] == 2)
+ request("/__release", {})
+ page.evaluate("Promise.all([window.first,window.second])")
+ complete(page)
+ assert page.evaluate("window.sim.documents.length") == 1, (
+ "Concurrent deduplication left duplicate chips"
+ )
+
+
+def materialize(page):
+ page.goto(BASE + "/?empty")
+ page.wait_for_selector("#state")
+ page.evaluate("window.sim.materialize()")
+ complete(page)
+
+
+CASES = [
+ ("file-" + ext, "normal", ordinary(ext))
+ for ext in ("pdf", "txt", "md", "markdown", "docx", "html", "htm")
+]
+CASES += [
+ ("duplicate-selection", "normal", reselect),
+ ("get-only-server", "get-only", ordinary("pdf")),
+ ("early-stream-end", "early-end", early_end),
+ ("unavailable-sse", "sse-unavailable", early_end),
+ ("indexing-error", "error", error),
+ ("upload-rejection", "reject", error),
+ ("chat-initialization-failure", "normal", failed_scope),
+ ("inflight-duplicate", "server-duplicate", duplicate_running),
+ ("navigate-during-post", "delayed-fail", navigate),
+ ("unmount-during-post", "delayed-fail", unmount),
+ ("twelve-files-open-streams", "hold", large_batch),
+ ("overlapping-uploads", "delayed", overlapping_uploads),
+ ("concurrent-same-content", "delayed-server-duplicate", concurrent_same_content),
+ ("lazy-thread-materialization", "normal", materialize),
+]
+
+
+def main():
+ records = []
+ engines = [arg for arg in sys.argv[1:] if arg != "--probe"] or ["chromium", "firefox", "webkit"]
+ with sync_playwright() as pw:
+ for engine in engines:
+ if engine == "safari":
+ from rag_upload_safari import SafariBrowser
+ browser = SafariBrowser(ROOT)
+ elif engine == "chrome":
+ browser = pw.chromium.launch(channel = "chrome", headless = True)
+ elif engine == "edge":
+ exe = next(
+ (ROOT / "edge-unpacked").rglob(
+ "Microsoft Edge.app/Contents/MacOS/Microsoft Edge"
+ ),
+ None,
+ )
+ browser = (
+ pw.chromium.launch(executable_path = str(exe), headless = True)
+ if exe
+ else pw.chromium.launch(channel = "msedge", headless = True)
+ )
+ else:
+ browser = getattr(pw, engine).launch(headless = True)
+ version = browser.version
+ if "--probe" in sys.argv:
+ browser.close()
+ continue
+ for name, mode, action in CASES:
+ start = time.monotonic()
+ try:
+ run_case(browser, mode, action)
+ record = {
+ "browser": engine,
+ "version": version,
+ "case": name,
+ "status": "passed",
+ }
+ except Exception as exc:
+ record = {
+ "browser": engine,
+ "version": version,
+ "case": name,
+ "status": "failed",
+ "error": str(exc),
+ }
+ record["seconds"] = round(time.monotonic() - start, 3)
+ print(json.dumps(record), flush = True)
+ records.append(record)
+ browser.close()
+ (ROOT / ("browser-results-" + "-".join(engines) + ".json")).write_text(
+ json.dumps(records, indent = 2)
+ )
+ raise SystemExit(any(r["status"] == "failed" for r in records))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/studio/rag_upload_ci.py b/tests/studio/rag_upload_ci.py
new file mode 100644
index 00000000000..2892f981dcf
--- /dev/null
+++ b/tests/studio/rag_upload_ci.py
@@ -0,0 +1,228 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Run isolated upload regressions on a native CI runner."""
+
+import json
+import os
+from pathlib import Path
+import platform
+import shutil
+import sqlite3
+import subprocess
+import sys
+import time
+import urllib.request
+
+repo = Path(__file__).resolve().parents[2]
+root = Path(os.environ.get("RAG_SIM_ART_DIR", str(repo / "temp/rag-upload"))).resolve()
+root.mkdir(parents = True, exist_ok = True)
+for folder in ("tmp", "cache", "browsers"):
+ (root / folder).mkdir(exist_ok = True)
+env = os.environ.copy()
+env.update(
+ {
+ "RAG_SIM_ART_DIR": str(root),
+ "TMPDIR": str(root / "tmp"),
+ "TMP": str(root / "tmp"),
+ "TEMP": str(root / "tmp"),
+ "UV_CACHE_DIR": str(root / "cache/uv"),
+ "PIP_CACHE_DIR": str(root / "cache/pip"),
+ "XDG_CACHE_HOME": str(root / "cache"),
+ "HF_HOME": str(root / "cache/huggingface"),
+ "PYTHONDONTWRITEBYTECODE": "1",
+ "PLAYWRIGHT_BROWSERS_PATH": str(root / "browsers"),
+ "npm_config_cache": str(root / "cache/npm"),
+ }
+)
+failures = []
+
+
+def run(
+ name,
+ args,
+ cwd = repo,
+ required = True,
+):
+ with (root / f"{name}.log").open("w", encoding = "utf-8") as log:
+ result = subprocess.run(
+ list(map(str, args)), cwd = cwd, env = env, stdout = log, stderr = subprocess.STDOUT
+ )
+ print(f"{name}: exit {result.returncode}", flush = True)
+ if result.returncode:
+ if required:
+ failures.append(name)
+ print(
+ (root / f"{name}.log").read_text(encoding = "utf-8", errors = "replace")[-6000:], flush = True
+ )
+ return result.returncode == 0
+
+
+def python_in(directory):
+ return directory / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
+
+
+assert run("bootstrap", [sys.executable, "-m", "venv", root / "bootstrap"])
+bootstrap = python_in(root / "bootstrap")
+assert run("install-uv", [bootstrap, "-m", "pip", "install", "uv==0.11.0"])
+# actions/setup-python ships macOS builds without --enable-loadable-sqlite-extensions, so
+# sqlite-vec cannot load and every RAG test errors. Download an interpreter that has one.
+# Read the path off the managed list rather than asking uv to resolve "3.12": both
+# `uv venv --managed-python` and `uv python find --managed-python` answered with the
+# runner's own framework build, while every path this list reports is one uv installed.
+if hasattr(sqlite3.connect(":memory:"), "enable_load_extension"):
+ interpreter = sys.executable
+else:
+ assert run("python-install", [bootstrap, "-m", "uv", "python", "install", "3.12"])
+ listed = subprocess.run(
+ [
+ str(bootstrap),
+ "-m",
+ "uv",
+ "python",
+ "list",
+ "--only-installed",
+ "--managed-python",
+ "--output-format",
+ "json",
+ ],
+ cwd = repo,
+ env = env,
+ capture_output = True,
+ text = True,
+ check = True,
+ ).stdout
+ interpreter = next(
+ entry["path"] for entry in json.loads(listed) if entry["version"].startswith("3.12.")
+ )
+assert run(
+ "venv", [bootstrap, "-m", "uv", "venv", "--clear", "--python", interpreter, root / "venv"]
+)
+python = python_in(root / "venv")
+# Fail here rather than through a hundred RagExtensionUnavailable errors.
+assert run(
+ "sqlite-extensions",
+ [python, "-c", "import sqlite3; sqlite3.connect(':memory:').enable_load_extension(True)"],
+)
+assert run(
+ "dependencies",
+ [
+ bootstrap,
+ "-m",
+ "uv",
+ "pip",
+ "install",
+ "--python",
+ python,
+ "-r",
+ repo / "tests/studio/rag_upload_requirements.txt",
+ ],
+)
+tests = sorted((repo / "studio/backend/tests").glob("test_rag_*.py"))
+tests.append(repo / "studio/backend/tests/test_conversation_archive.py")
+run(
+ "backend",
+ [
+ python,
+ repo / "tests/studio/rag_upload_pytest.py",
+ *tests,
+ "-q",
+ "-rs",
+ "--basetemp",
+ root / "pytest",
+ "-o",
+ f"cache_dir={root / 'pytest-cache'}",
+ f"--junitxml={root / 'backend.xml'}",
+ ],
+)
+frontend = repo / "studio/frontend"
+npm = shutil.which("npm.cmd" if os.name == "nt" else "npm")
+if run("frontend-install", [npm, "ci"], cwd = frontend):
+ node_tests = sorted(
+ set(
+ p
+ for pattern in (
+ "rag-*.test.ts",
+ "project-attachment*.test.ts",
+ "project-source*.test.ts",
+ )
+ for p in (frontend / "tests").glob(pattern)
+ )
+ )
+ run(
+ "frontend-unit", ["node", "--experimental-strip-types", "--test", *node_tests], cwd = frontend
+ )
+ run("typecheck", [npm, "run", "typecheck"], cwd = frontend)
+ if run(
+ "browser-install", [python, "-m", "playwright", "install", "chromium", "firefox", "webkit"]
+ ):
+ engines = ["chromium", "firefox", "webkit"]
+ driver = repo / "tests/studio/rag_upload_browser_tests.py"
+ if not run("browser-probe", [python, driver, "--probe", *engines], required = False):
+ if sys.platform == "linux":
+ env.update(
+ {
+ "RETRY_ATTEMPTS": "2",
+ "RETRY_ATTEMPT_TIMEOUT": "180",
+ "APT_ACQUIRE_RETRIES": "0",
+ }
+ )
+ run(
+ "browser-libraries",
+ [
+ "bash",
+ repo / ".github/scripts/retry-with-apt-lock.sh",
+ python,
+ "-m",
+ "playwright",
+ "install-deps",
+ *engines,
+ ],
+ )
+ if os.name == "nt":
+ engines += ["chrome", "edge"]
+ if sys.platform == "darwin" and env.get("GITHUB_ACTIONS") == "true":
+ if run("safari-enable", ["sudo", "/usr/bin/safaridriver", "--enable"]):
+ engines.append("safari")
+ with (root / "browser-server.log").open("w", encoding = "utf-8") as log:
+ server = subprocess.Popen(
+ [
+ "node",
+ str(repo / "tests/studio/rag_upload_browser_server.mjs"),
+ str(frontend),
+ str(root),
+ ],
+ cwd = repo,
+ env = env,
+ stdout = log,
+ stderr = subprocess.STDOUT,
+ )
+ try:
+ deadline = time.monotonic() + 30
+ while True:
+ if server.poll() is not None:
+ raise RuntimeError("Browser fixture exited before becoming ready")
+ try:
+ urllib.request.urlopen("http://127.0.0.1:18948/__state", timeout = 1).close()
+ break
+ except OSError:
+ if time.monotonic() > deadline:
+ raise RuntimeError("Browser fixture did not start")
+ time.sleep(0.1)
+ run(
+ "browsers",
+ [python, driver, *engines],
+ )
+ finally:
+ server.terminate()
+ try:
+ server.wait(timeout = 10)
+ except subprocess.TimeoutExpired:
+ server.kill()
+ server.wait()
+(root / "summary.json").write_text(
+ json.dumps(
+ {"os": platform.platform(), "python": sys.version, "failed_steps": failures}, indent = 2
+ )
+)
+raise SystemExit(bool(failures))
diff --git a/tests/studio/rag_upload_pytest.py b/tests/studio/rag_upload_pytest.py
new file mode 100644
index 00000000000..6a155f9a602
--- /dev/null
+++ b/tests/studio/rag_upload_pytest.py
@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Keep test homes and caches inside the simulation directory."""
+
+import os
+from pathlib import Path
+import sys
+
+root = Path(os.environ.get("RAG_SIM_ART_DIR", "./temp/rag-upload")).resolve()
+root.mkdir(parents = True, exist_ok = True)
+test_home = root / "home"
+test_home.mkdir(exist_ok = True)
+Path.home = classmethod(lambda cls: test_home)
+expanduser = os.path.expanduser
+os.path.expanduser = lambda path: (
+ str(test_home) + path[1:]
+ if isinstance(path, str) and (path == "~" or path.startswith(("~/", "~\\")))
+ else expanduser(path)
+)
+os.environ.update(
+ {
+ "UNSLOTH_STUDIO_HOME": str(test_home / "studio"),
+ "XDG_CACHE_HOME": str(root / "cache"),
+ "HF_HOME": str(root / "cache/huggingface"),
+ "HYPOTHESIS_STORAGE_DIRECTORY": str(root / "hypothesis"),
+ }
+)
+import pytest
+
+raise SystemExit(pytest.main(sys.argv[1:]))
diff --git a/tests/studio/rag_upload_requirements.txt b/tests/studio/rag_upload_requirements.txt
new file mode 100644
index 00000000000..ae25fce7d83
--- /dev/null
+++ b/tests/studio/rag_upload_requirements.txt
@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+annotated-doc==0.0.5
+annotated-types==0.8.0
+anyio==4.15.1
+argon2-cffi==25.1.0
+argon2-cffi-bindings==26.1.0
+beautifulsoup4==4.15.0
+certifi==2026.7.22
+cffi==2.1.1
+charset-normalizer==3.5.1
+click==8.5.0
+cryptography==50.0.1
+fastapi==0.141.1
+filelock==3.32.5
+fsspec==2026.7.0
+h11==0.16.0
+hf-xet==1.6.0
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==1.30.0
+idna==3.19
+iniconfig==2.3.0
+lxml==6.1.3
+markdown-it-py==4.2.0
+mdurl==0.1.2
+numpy==2.5.3
+packaging==26.3
+pluggy==1.6.0
+psutil==7.2.2
+pycparser==3.0
+pydantic==2.13.5
+pydantic-core==2.46.5
+pygments==2.21.0
+pyjwt==2.13.0
+pymupdf==1.27.2.3
+pymupdf4llm==0.3.4
+pytest==9.1.1
+python-docx==1.2.0
+python-multipart==0.0.32
+pyyaml==6.0.3
+requests==2.34.2
+rich==15.0.0
+ruff==0.6.9
+safetensors==0.8.0
+soupsieve==2.9.2
+sqlite-vec==0.1.9
+starlette==1.6.0
+structlog==26.1.0
+tabulate==0.10.0
+tqdm==4.70.0
+typing-extensions==4.16.0
+typing-inspection==0.4.4
+urllib3==2.7.0
+hypothesis==6.167.1
+playwright==1.62.0
+selenium==4.48.0
diff --git a/tests/studio/rag_upload_safari.py b/tests/studio/rag_upload_safari.py
new file mode 100644
index 00000000000..fc888a84004
--- /dev/null
+++ b/tests/studio/rag_upload_safari.py
@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Run the upload scenarios through native Safari WebDriver."""
+
+import time
+from pathlib import Path
+
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+
+
+class SafariBrowser:
+ def __init__(self, root):
+ self.driver = webdriver.Safari()
+ self.driver.set_script_timeout(15)
+ self.version = self.driver.capabilities.get("browserVersion", "unknown")
+ self.root = Path(root)
+
+ def new_context(self):
+ return SafariContext(self.driver, self.root)
+
+ def close(self):
+ self.driver.quit()
+
+
+class SafariContext:
+ def __init__(self, driver, root):
+ self.driver, self.root = driver, root
+
+ def new_page(self):
+ return SafariPage(self.driver, self.root)
+
+ def close(self):
+ self.driver.get("about:blank")
+ self.driver.delete_all_cookies()
+
+
+class SafariPage:
+ def __init__(self, driver, root):
+ self.driver, self.root = driver, root
+
+ def on(self, event, callback):
+ # The shared fixture records page errors for Safari.
+ pass
+
+ def goto(self, url):
+ self.driver.get(url)
+
+ def evaluate(self, expression):
+ result = self.driver.execute_async_script(
+ "const done=arguments[arguments.length-1];"
+ "Promise.resolve().then(()=>window.eval(arguments[0]))"
+ ".then(value=>done({value}),error=>done({error:String(error)}));",
+ expression,
+ )
+ if "error" in result:
+ raise AssertionError(result["error"])
+ return result.get("value")
+
+ def wait_for_function(
+ self,
+ expression,
+ timeout = 8000,
+ ):
+ WebDriverWait(self.driver, timeout / 1000, poll_frequency = 0.05).until(
+ lambda _: self.evaluate(expression)
+ )
+
+ def wait_for_selector(self, selector):
+ WebDriverWait(self.driver, 8).until(
+ lambda driver: driver.find_element(By.CSS_SELECTOR, selector)
+ )
+
+ def wait_for_timeout(self, milliseconds):
+ time.sleep(milliseconds / 1000)
+
+ def locator(self, selector):
+ return SafariLocator(self.driver, self.root, selector)
+
+
+class SafariLocator:
+ def __init__(self, driver, root, selector):
+ self.driver, self.root, self.selector = driver, root, selector
+
+ def click(self):
+ self.driver.find_element(By.CSS_SELECTOR, self.selector).click()
+
+ def set_input_files(self, file):
+ directory = self.root / "safari-files"
+ directory.mkdir(exist_ok = True)
+ path = directory / file["name"]
+ path.write_bytes(file["buffer"])
+ self.driver.find_element(By.CSS_SELECTOR, self.selector).send_keys(str(path))