Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep ticked PDF form check boxes in knowledge bases (#15572)
Nilaydb9d4f018ee4
diff --git a/studio/backend/core/rag/parsers.py b/studio/backend/core/rag/parsers.pyindex c79e409cf5f..9a0a3cafc6d 100644--- a/studio/backend/core/rag/parsers.py+++ b/studio/backend/core/rag/parsers.py@@ -436,6 +436,51 @@ def _pdf_markdown(doc, pages: range | None = None) -> list[str] | None:return [str(c.get("text") or "") for c in chunks]+def _pdf_form_field_lines(page) -> list[str]:+ import fitz++ try:+ widgets = list(page.widgets())+ except Exception as exc: # noqa: BLE001 - a malformed field must not fail the whole PDF+ logger.warning("skipping form fields on PDF page %d: %s", page.number + 1, exc)+ return []+ tooltip_owners: dict[str, set[str]] = {}+ for widget in widgets:+ tooltip = " ".join((widget.field_label or "").split())+ tooltip_owners.setdefault(tooltip, set()).add(widget.field_name)+ fields = {}+ for widget in widgets:+ shown = dict(c for c in widget.choice_values or [] if isinstance(c, (list, tuple)))+ value = widget.field_value+ value = ", ".join(+ shown.get(part, part)+ for part in (value if isinstance(value, list) else [value])+ if isinstance(part, str)+ )+ radio = widget.field_type == fitz.PDF_WIDGET_TYPE_RADIOBUTTON+ # Kids of one field share its /V; the widget's own /AS says which kid is ticked.+ unchecked = (radio or widget.field_type == fitz.PDF_WIDGET_TYPE_CHECKBOX) and (+ value == "Off" or page.parent.xref_get_key(widget.xref, "AS") == ("name", "/Off")+ )+ password = (+ widget.field_type == fitz.PDF_WIDGET_TYPE_TEXT+ and widget.field_flags & fitz.PDF_TX_FIELD_IS_PASSWORD+ )+ _, flags = page.parent.xref_get_key(widget.xref, "F")+ hidden = widget.field_display in (1, 3) or (+ flags.isdigit() and int(flags) & fitz.PDF_ANNOT_IS_INVISIBLE+ )+ if not widget.field_name or not value.strip() or unchecked or password or hidden:+ continue+ # A radio widget's /TU tooltip names one option, not the group's selected value.+ tooltip = "" if radio else " ".join((widget.field_label or "").split())+ label = tooltip or widget.field_name+ if tooltip and len(tooltip_owners[tooltip]) > 1:+ label = f"{tooltip} ({widget.field_name})"+ fields[widget.field_name] = f"{label}: {value}"+ return list(fields.values())++def _pdf(source: str | bytes,want_images: bool,@@ -453,6 +498,10 @@ def _pdf(raise ValueError("encrypted PDF requires a password")total_pages = doc.page_countpage_numbers = range(total_pages if max_pages is None else min(total_pages, max_pages))+ # Read before pymupdf4llm, which bakes the widgets into page content.+ form_fields = (+ {n: _pdf_form_field_lines(doc[n]) for n in page_numbers} if doc.is_form_pdf else {}+ )if not config.PDF_MARKDOWN:md = Noneelif max_pages is None:@@ -491,6 +540,7 @@ def _pdf(breakif needs_ocr:text = plain+ text = "\n".join(filter(None, [text, *form_fields.get(page_number, [])]))pages.append(Page(text, page_number + 1, len(text), needs_ocr = needs_ocr))if want_images:for img in page.get_images(full = True):diff --git a/studio/backend/tests/test_rag_parsing.py b/studio/backend/tests/test_rag_parsing.pyindex 43dd881e75a..35b9b80e11d 100644--- a/studio/backend/tests/test_rag_parsing.py+++ b/studio/backend/tests/test_rag_parsing.py@@ -165,6 +165,134 @@ def test_pdf_markdown_off_uses_plain_text(tmp_path, monkeypatch):assert "#" not in text and "|" not in text[email protected]("markdown", [True, False])+def test_pdf_form_keeps_which_boxes_are_ticked(tmp_path, monkeypatch, markdown):+ if markdown:+ pytest.importorskip("pymupdf4llm")+ import pymupdf++ from core.rag import config, parsers++ monkeypatch.setattr(config, "PDF_MARKDOWN", markdown)+ doc = pymupdf.open()+ page = doc.new_page()+ page.insert_text((40, 60), "Smoker:")+ for x, answer, ticked in ((120, "Yes", False), (180, "No", True)):+ widget = pymupdf.Widget()+ widget.field_type = pymupdf.PDF_WIDGET_TYPE_CHECKBOX+ widget.field_name = f"smoker_{answer.lower()}"+ widget.field_label = f"Smoker {answer}"+ widget.rect = pymupdf.Rect(x, 50, x + 12, 62)+ widget.field_value = ticked+ page.add_widget(widget)+ page.insert_text((x + 16, 60), answer)+ pdf = tmp_path / "form.pdf"+ doc.save(str(pdf))+ doc.close()++ text = "\n".join(p.text for p in parsers.parse(str(pdf)))+ assert "Smoker No: Yes" in text+ assert "Smoker Yes" not in text++[email protected]("shared_field", [True, False])+def test_pdf_form_names_the_ticked_box_when_boxes_share_a_name_or_tooltip(tmp_path, shared_field):+ import pymupdf++ from core.rag import parsers++ doc = pymupdf.open()+ page = doc.new_page()+ refs = []+ for x, answer in ((120, "Yes"), (210, "No")):+ widget = pymupdf.Widget()+ widget.field_type = pymupdf.PDF_WIDGET_TYPE_CHECKBOX+ widget.field_name = f"smoker_{answer.lower()}"+ widget.field_label = f"Smoker {answer}" if shared_field else "Smoker"+ widget.field_value = answer == ("Yes" if shared_field else "No")+ widget.rect = pymupdf.Rect(x, 50, x + 12, 62)+ refs.append(page.add_widget(widget).xref)+ if shared_field:+ # One /Btn field, two kids with export values Yes / No; both inherit /V /Yes.+ parent = doc.get_new_xref()+ doc.update_object(+ parent, f"<< /FT /Btn /T (smoker) /V /Yes /Kids [{refs[0]} 0 R {refs[1]} 0 R] >>"+ )+ for ref in refs:+ doc.xref_set_key(ref, "Parent", f"{parent} 0 R")+ for key in ("T", "FT", "V"):+ doc.xref_set_key(ref, key, "null")+ _, on = doc.xref_get_key(refs[1], "AP/N/Yes")+ _, off = doc.xref_get_key(refs[1], "AP/N/Off")+ doc.xref_set_key(refs[1], "AP/N", f"<< /Off {off} /No {on} >>")+ doc.xref_set_key(doc.pdf_catalog(), "AcroForm/Fields", f"[{parent} 0 R]")+ pdf = tmp_path / "form.pdf"+ doc.save(str(pdf))+ doc.close()++ with pymupdf.open(str(pdf)) as reopened:+ lines = parsers._pdf_form_field_lines(reopened[0])+ if shared_field:+ # pymupdf 2.x labels the kid with its parent field ("smoker: Yes").+ assert len(lines) == 1 and lines[0].endswith(": Yes") and "No" not in lines[0]+ else:+ assert lines == ["Smoker (smoker_no): Yes"]++[email protected]("flag", [1, 2, 32])+def test_pdf_form_skips_invisible_hidden_and_noview_fields(tmp_path, flag):+ import pymupdf++ from core.rag import parsers++ doc = pymupdf.open()+ page = doc.new_page()+ for name, y in (("shown", 80), ("masked", 120)):+ widget = pymupdf.Widget()+ widget.field_type = pymupdf.PDF_WIDGET_TYPE_TEXT+ widget.field_name = name+ widget.field_value = f"{name} value"+ widget.rect = pymupdf.Rect(40, y, 240, y + 20)+ xref = page.add_widget(widget).xref+ doc.xref_set_key(xref, "F", str(flag))+ pdf = tmp_path / "form.pdf"+ doc.save(str(pdf))+ doc.close()++ with pymupdf.open(str(pdf)) as reopened:+ assert parsers._pdf_form_field_lines(reopened[0]) == ["shown: shown value"]+++# PyMuPDF raises while loading either widget; main never read widgets, so it parsed these.[email protected]("key, value", [("DA", "(garbage Tf)"), ("Parent", "self")])[email protected]("markdown", [True, False])+def test_pdf_with_a_malformed_form_field_still_reads_its_text(+ tmp_path, monkeypatch, markdown, key, value+):+ if markdown:+ pytest.importorskip("pymupdf4llm")+ import pymupdf++ from core.rag import config, parsers++ monkeypatch.setattr(config, "PDF_MARKDOWN", markdown)+ doc = pymupdf.open()+ page = doc.new_page()+ page.insert_text((40, 60), "Budget: 4200")+ widget = pymupdf.Widget()+ widget.field_type = pymupdf.PDF_WIDGET_TYPE_TEXT+ widget.field_name = "notes"+ widget.field_value = "x"+ widget.rect = pymupdf.Rect(40, 80, 240, 100)+ xref = page.add_widget(widget).xref+ doc.xref_set_key(xref, key, f"{xref} 0 R" if value == "self" else value)+ pdf = tmp_path / "damaged.pdf"+ doc.save(str(pdf))+ doc.close()++ assert "Budget: 4200" in "\n".join(p.text for p in parsers.parse(str(pdf)))++def test_pdf_bytes_use_same_extraction_path(tmp_path, monkeypatch):from core.rag import config, parsers