Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read each PDF page's words once when locating chunk regions (#15665)
Daniel Hana061fb19e64e
diff --git a/studio/backend/core/rag/locators.py b/studio/backend/core/rag/locators.pyindex 384b54dd9b5..279156a41ff 100644--- a/studio/backend/core/rag/locators.py+++ b/studio/backend/core/rag/locators.py@@ -37,8 +37,14 @@ def _anchor_tokens(page_text: str, match: LocatorMatch) -> list[str]:raw = segment.replace("|", " ").split()if len(raw) >= MIN_ANCHOR_WORDS + 2:raw = raw[1:-1]- tokens = [t for t in (_norm_token(w) for w in raw) if t]- return tokens[:MAX_ANCHOR_WORDS]+ tokens: list[str] = []+ for w in raw:+ t = _norm_token(w)+ if t:+ tokens.append(t)+ if len(tokens) == MAX_ANCHOR_WORDS:+ break+ return tokensdef _find_subsequences(haystack: list[str], needle: list[str]) -> list[int]:@@ -54,8 +60,8 @@ def _find_subsequences(haystack: list[str], needle: list[str]) -> list[int]:return out-def _locate(page_words: list, needle: list[str]) -> list[int] | None:- """Skip punctuation-only words so they never break a phrase."""+def _page_tokens(page_words: list) -> tuple[list[str], list[int]]:+ """Normalized tokens (punctuation-only words skipped) and their ``page_words`` indices."""tokens: list[str] = []idx_map: list[int] = []for j, w in enumerate(page_words):@@ -63,7 +69,15 @@ def _locate(page_words: list, needle: list[str]) -> list[int] | None:if t:tokens.append(t)idx_map.append(j)+ return tokens, idx_map++def _locate(+ page_words: list,+ needle: list[str],+ page_tokens: tuple[list[str], list[int]] | None = None,+) -> list[int] | None:+ tokens, idx_map = page_tokens if page_tokens is not None else _page_tokens(page_words)ambiguous_first: list[int] | None = Nonefor size in range(len(needle), MIN_ANCHOR_WORDS - 1, -1):sub = needle[:size]@@ -108,22 +122,59 @@ def _rects_from_words(page_words: list, indices: list[int], pw: float, ph: floatreturn out-def _regions_for_match(doc: Any, page_text: str, match: LocatorMatch) -> list[dict[str, Any]]:+class _PageMemo:+ """One-page cache of words, tokens and size: chunks arrive in page order. Failed+ reads are not cached, so the next chunk retries them."""++ __slots__ = ("index", "page", "words", "tokens", "size")++ def __init__(self) -> None:+ self.index = None+ self.page = None+ self.words = None+ self.tokens = None+ self.size = None++ def words_for(self, doc: Any, index: int):+ if self.index != index:+ self.index = self.page = self.words = self.tokens = self.size = None+ page = doc[index]+ words = page.get_text("words") or []+ self.index, self.page, self.words = index, page, words+ return self.page, self.words++ def tokens_for(self, words: list) -> tuple[list[str], list[int]]:+ if self.tokens is None:+ self.tokens = _page_tokens(words)+ return self.tokens++ def size_for(self, page: Any) -> tuple[float, float]:+ if self.size is None:+ self.size = (float(page.rect.width), float(page.rect.height))+ return self.size+++def _regions_for_match(+ doc: Any,+ page_text: str,+ match: LocatorMatch,+ memo: _PageMemo | None = None,+) -> list[dict[str, Any]]:try:if match.page_index < 0 or match.page_index >= len(doc):return []needle = _anchor_tokens(page_text, match)if len(needle) < MIN_ANCHOR_WORDS:return []- page = doc[match.page_index]- page_words = page.get_text("words") or []+ if memo is None:+ memo = _PageMemo()+ page, page_words = memo.words_for(doc, match.page_index)if not page_words:return []- indices = _locate(page_words, needle)+ indices = _locate(page_words, needle, memo.tokens_for(page_words))if not indices:return []- pw = float(page.rect.width)- ph = float(page.rect.height)+ pw, ph = memo.size_for(page)if pw <= 0 or ph <= 0:return []rects = _rects_from_words(page_words, indices, pw, ph)@@ -149,6 +200,7 @@ def pdf_regions_for_chunks(pdf_path: Path, pages: list, chunks: list) -> list[lireturn [[] for _ in chunks]regions: list[list[dict[str, Any]]] = []+ memo = _PageMemo()try:for chunk in chunks:page_index = getattr(chunk, "source_page_index", None)@@ -166,7 +218,7 @@ def pdf_regions_for_chunks(pdf_path: Path, pages: list, chunks: list) -> list[listart = int(start),end = int(end),)- regions.append(_regions_for_match(doc, pages[page_index].text, match))+ regions.append(_regions_for_match(doc, pages[page_index].text, match, memo))return regionsfinally:doc.close()diff --git a/studio/backend/tests/test_rag_locator_page_memo.py b/studio/backend/tests/test_rag_locator_page_memo.pynew file mode 100644index 00000000000..65bdd4cb4ef--- /dev/null+++ b/studio/backend/tests/test_rag_locator_page_memo.py@@ -0,0 +1,129 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""pdf_regions_for_chunks reads each page once per run and matches per-chunk location."""++import random++import pytest++pymupdf = pytest.importorskip("pymupdf")++from core.rag import locators+from core.rag.chunking import chunk_pages+from core.rag.locators import (+ MAX_ANCHOR_WORDS,+ LocatorMatch,+ _anchor_tokens,+ _norm_token,+ _regions_for_match,+)+from core.rag.parsers import parse_pdf_bytes++_WORDS = [+ "alpha", "beta", "gamma", "delta", "significant", "effort", "Revenue", "Q1",+ "growth", "model", "data", "**bold**", "(paren)", "end.", "the", "of", "and", "cell|row",+] # fmt: skip+++def _pdf(tmp_path):+ """Five dense pages: page 1 is empty, page 3 is rotated."""+ rng = random.Random(7)+ doc = pymupdf.open()+ for i in range(5):+ page = doc.new_page()+ if i == 1:+ continue+ for y in range(60, 780, 12):+ page.insert_text((50, y), " ".join(rng.choice(_WORDS) for _ in range(12)), fontsize = 9)+ if i == 3:+ page.set_rotation(90)+ path = tmp_path / "memo.pdf"+ doc.save(str(path))+ doc.close()+ pages, _ = parse_pdf_bytes(path.read_bytes())+ chunks = chunk_pages(pages, max_tokens = 64, overlap = 8, count = lambda t: len(t.split()))+ return path, pages, chunks+++def _one_by_one(path, pages, chunks):+ """Each chunk located on a freshly opened page, as before the memo."""+ doc = pymupdf.open(str(path))+ try:+ return [+ _regions_for_match(+ doc,+ pages[c.source_page_index].text,+ LocatorMatch(+ c.source_page_index, c.page_number, c.page_char_start, c.page_char_end+ ),+ )+ for c in chunks+ ]+ finally:+ doc.close()+++def get_text_calls(monkeypatch):+ calls = []+ original = pymupdf.Page.get_text++ def counting(self, *args, **kwargs):+ calls.append(self.number)+ return original(self, *args, **kwargs)++ monkeypatch.setattr(pymupdf.Page, "get_text", counting)+ return calls+++def test_regions_match_per_chunk_location_in_page_order(tmp_path, get_text_calls):+ path, pages, chunks = _pdf(tmp_path)+ expected = _one_by_one(path, pages, chunks)+ assert sum(1 for r in expected if r) > len(chunks) // 2+ del get_text_calls[:]+ assert locators.pdf_regions_for_chunks(path, pages, chunks) == expected+ assert get_text_calls == sorted({c.source_page_index for c in chunks})+++def test_regions_match_when_pages_interleave(tmp_path, get_text_calls):+ """A page revisited after another page must not reuse the other page's words."""+ path, pages, chunks = _pdf(tmp_path)+ shuffled = list(chunks)+ random.Random(1).shuffle(shuffled)+ expected = _one_by_one(path, pages, shuffled)+ assert locators.pdf_regions_for_chunks(path, pages, shuffled) == expected+++def test_a_failed_page_read_is_retried_for_the_next_chunk(tmp_path, monkeypatch):+ path, pages, chunks = _pdf(tmp_path)+ expected = _one_by_one(path, pages, chunks)+ first_page = [i for i, c in enumerate(chunks) if c.source_page_index == 0]+ assert len(first_page) >= 3 and expected[first_page[0]] and expected[first_page[1]]+ original = pymupdf.Page.get_text+ state = {"failed": False}++ def flaky(self, *args, **kwargs):+ if not state["failed"]:+ state["failed"] = True+ raise RuntimeError("transient")+ return original(self, *args, **kwargs)++ monkeypatch.setattr(pymupdf.Page, "get_text", flaky)+ got = locators.pdf_regions_for_chunks(path, pages, chunks)+ assert got[first_page[0]] == []+ assert got[first_page[1] :] == expected[first_page[1] :]+++def test_anchor_tokens_equal_the_full_normalized_prefix():+ rng = random.Random(3)+ for _ in range(200):+ text = " ".join(+ rng.choice(_WORDS + ["...", "--", "|", "—"]) for _ in range(rng.randint(0, 40))+ )+ match = LocatorMatch(0, 1, 0, len(text))+ raw = text.replace("|", " ").split()+ if len(raw) >= 6:+ raw = raw[1:-1]+ full = [t for t in (_norm_token(w) for w in raw) if t]+ assert _anchor_tokens(text, match) == full[:MAX_ANCHOR_WORDS]