Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: decode escaped spreadsheet text during RAG indexing (#16712)
DimitriChrysafis7ac3ea5a1006
diff --git a/studio/backend/core/rag/office_formats.py b/studio/backend/core/rag/office_formats.pyindex 198509bcbfe..5ab70eda5cd 100644--- a/studio/backend/core/rag/office_formats.py+++ b/studio/backend/core/rag/office_formats.py@@ -491,16 +491,32 @@ def _column_index(ref: str) -> int:return index - 1+def _decode_xstring(text: str) -> str:+ if "_x" not in text:+ return text+ return re.sub(+ r"_x005[Ff]_(x[0-9a-fA-F]{4}_)|_x([0-9a-fA-F]{4})_",+ lambda match: f"_{match[1]}" if match[1] else chr(int(match[2], 16)),+ text,+ )+++def _xlsx_unicode(text: str) -> str:+ if re.search(r"[\ud800-\udfff]", text):+ return text.encode("utf-16-le", "surrogatepass").decode("utf-16-le", "replace")+ return text++def _xlsx_text(node: ET.Element) -> str:# Plain <t> or rich runs <r><t>; phonetic hints (<rPh>) are skipped.parts = []for child in node:if child.tag == _q("s", "t"):- parts.append(child.text or "")+ parts.append(_decode_xstring(child.text or ""))elif child.tag == _q("s", "r"):t = child.find(_q("s", "t"))- parts.append((t.text or "") if t is not None else "")- return "".join(parts)+ parts.append(_decode_xstring(t.text or "") if t is not None else "")+ return _xlsx_unicode("".join(parts))def xlsx(path: str) -> list[Section]:@@ -562,7 +578,9 @@ def xlsx(path: str) -> list[Section]:value = _xlsx_text(inline) if inline is not None else ""elif kind == "b":value = "TRUE" if raw == "1" else "FALSE"- elif kind in ("str", "e"):+ elif kind == "str":+ value = _xlsx_unicode(_decode_xstring(raw))+ elif kind == "e":value = rawelse:try:diff --git a/studio/backend/tests/test_rag_office_formats.py b/studio/backend/tests/test_rag_office_formats.pyindex 18c4388811c..31a7f9765bf 100644--- a/studio/backend/tests/test_rag_office_formats.py+++ b/studio/backend/tests/test_rag_office_formats.py@@ -530,6 +530,80 @@ def test_xlsx_reads_shared_inline_dates_and_gaps(tmp_path):)+def _xlsx_strings(path, runs, cached):+ return _zip(+ path,+ {+ "xl/workbook.xml": f'<workbook {S}><sheets><sheet name="Text" sheetId="1" r:id="rId1"/></sheets></workbook>',+ "xl/_rels/workbook.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="worksheets/sheet1.xml"/></Relationships>',+ "xl/sharedStrings.xml": f"<sst {S}><si>{runs}</si></sst>",+ "xl/worksheets/sheet1.xml": f'<worksheet {S}><sheetData><row r="1">'+ '<c r="A1" t="s"><v>0</v></c>'+ f'<c r="B1" t="inlineStr"><is>{runs}</is></c>'+ f'<c r="C1" t="str"><f>A1</f><v>{cached}</v></c>'+ '<c r="D1"><v>42</v></c><c r="E1" t="e"><v>#VALUE!</v></c>'+ "</row></sheetData></worksheet>",+ },+ )++[email protected]("strict", [False, True])+ "encoded, expected",+ [+ ("First_x000D_\nSecond_x0009_End", "First\r\nSecond\tEnd"),+ ("Smile _xD83D__xDE00_", "Smile \U0001f600"),+ ("_x005F_x000D_", "_x000D_"),+ ("_x005f_x005F_x000D_", "_x005F_x000D_"),+ ("&_x0041_ _xZZZZ_", "&A _xZZZZ_"),+ ("_X0000_ _x00aB_ _x005F_x0041_", "_X0000_ \u00ab _x0041_"),+ ("A_xD83D_B_xDE00_C", "A\ufffdB\ufffdC"),+ ("plain text", "plain text"),+ ],+)+def test_xlsx_decodes_escaped_cell_strings(tmp_path, strict, encoded, expected):+ path = _xlsx_strings(tmp_path / "text.xlsx", f"<t>{encoded}</t>", encoded)+ if strict:+ path = _strict(path, tmp_path / "strict.xlsx")+ assert office_formats.xlsx(str(path)) == [+ (f"Sheet: Text\n{expected} | {expected} | {expected} | 42 | #VALUE!", None)+ ]+++def test_xlsx_decodes_each_run_before_joining_surrogate_pairs(tmp_path):+ runs = (+ "<r><t>_x00</t></r><r><t>0D_</t></r>"+ "<r><t>_xD83D_</t></r><r><t>_xDE00_</t></r>"+ "<rPh><t>ignored_x0041_</t></rPh>"+ )+ path = _xlsx_strings(tmp_path / "rich.xlsx", runs, "cached")+ assert (+ _text(path) == "Sheet: Text\n_x000D_\U0001f600 | _x000D_\U0001f600 | cached | 42 | #VALUE!"+ )++[email protected]("extension", [".xlsx", ".xlsm", ".xltx", ".xltm"])+def test_escaped_xlsx_uploads_are_searchable(rag_home, stub_embeddings, tmp_path, extension):+ if not rag_db.rag_available():+ pytest.skip("sqlite-vec unavailable here")+ encoded = "Zebra_x006D_arker _xD83D__xDE00_"+ source = _xlsx_strings(tmp_path / f"escaped{extension}", f"<t>{encoded}</t>", encoded)+ stored, filename, _hash = _save_upload(+ UploadFile(file = io.BytesIO(source.read_bytes()), filename = source.name)+ )+ scope = store.thread_scope("escaped-office")+ doc_id, job = ingestion.start_ingestion(+ scope, None, "escaped-office", filename, stored, background = False+ )+ assert ingestion.get_job_status(job)["status"] == "completed"+ conn = rag_db.get_connection()+ try:+ assert store.get_document(conn, doc_id)["status"] == "completed"+ assert store.search_lexical(conn, scope, "zebramarker", 5)+ finally:+ conn.close()++def test_pptx_reads_slides_in_order_with_tables_and_notes(tmp_path):pages = parsers.parse(str(build_pptx(tmp_path / "deck.pptx")))assert [(p.page_number, p.text) for p in pages] == [