Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio RAG: read each spreadsheet number format once per workbook (#16339)
Daniel Han02372492c33e
diff --git a/studio/backend/core/rag/office_formats.py b/studio/backend/core/rag/office_formats.pyindex 576aebad406..afd24a878d8 100644--- a/studio/backend/core/rag/office_formats.py+++ b/studio/backend/core/rag/office_formats.py@@ -322,10 +322,12 @@ def _group(digits: str) -> str:return ",".join([digits[:head]] + [digits[i : i + 3] for i in range(head, len(digits), 3)])-def _format_section(value: float, tokens: list[tuple[str, str]]) -> str | None:- marks = [i for i, (kind, _) in enumerate(tokens) if kind in ("ph", "exp")]+def _section_plan(tokens: list[tuple[str, str]]):+ """The value-independent reading of one format section: its literal text when it shows no+ digits, None when unsupported, else the layout ``_apply_section`` fills in."""if not any(kind == "ph" and ch in "0#?" for kind, ch in tokens):return "".join(text for kind, text in tokens if kind == "lit")+ marks = [i for i, (kind, _) in enumerate(tokens) if kind in ("ph", "exp")]first, last = marks[0], marks[-1]if any(kind == "lit" for kind, _ in tokens[first:last]):return None@@ -333,24 +335,60 @@ def _format_section(value: float, tokens: list[tuple[str, str]]) -> str | None:suffix = "".join(text for kind, text in tokens[last + 1 :] if kind == "lit")pattern = "".join(ch if kind == "ph" else "E" + ch for kind, ch in tokens[first : last + 1])percent = "%" * pattern.count("%")- value *= 100 ** len(percent)pattern = pattern.replace("%", "")mantissa, _, exponent = pattern.partition("E")sign_style, exponent = (exponent[:1], exponent[1:]) if exponent else ("", "")whole, dot, fraction = mantissa.partition(".")+ thousands = 0while whole.endswith(","): # trailing commas scale by thousands- whole, value = whole[:-1], value / 1000+ whole, thousands = whole[:-1], thousands + 1grouped, whole = "," in whole, whole.replace(",", "")places, required = sum(fraction.count(c) for c in "0#?"), fraction.count("0")- min_whole = whole.count("0")+ return (+ prefix,+ suffix,+ percent,+ thousands,+ grouped,+ dot,+ places,+ required,+ whole.count("0"),+ sign_style,+ max(len(whole), 1),+ "#" in whole,+ exponent.count("0"),+ )+++def _apply_section(value: float, plan) -> str | None:+ if plan is None or isinstance(plan, str):+ return plan+ (+ prefix,+ suffix,+ percent,+ thousands,+ grouped,+ dot,+ places,+ required,+ min_whole,+ sign_style,+ width,+ hashed,+ exp_digits,+ ) = plan+ value *= 100 ** len(percent)+ for _ in range(thousands):+ value = value / 1000power = 0if sign_style:- width = max(len(whole), 1)if value:power = math.floor(math.log10(value))- power = power - power % width if "#" in whole else power - (max(min_whole, 1) - 1)+ power = power - power % width if hashed else power - (max(min_whole, 1) - 1)if float(_round(value / 10**power, places)) >= 10 ** max(width, min_whole, 1):- power += width if "#" in whole else 1+ power += width if hashed else 1value /= 10**powerdigits = _round(value, places)int_part, _, frac_part = digits.partition(".")@@ -362,12 +400,16 @@ def _format_section(value: float, tokens: list[tuple[str, str]]) -> str | None:text = int_part + (dot + frac_part if dot else "")if sign_style:exp_sign = "-" if power < 0 else ("+" if sign_style == "+" else "")- text += f"E{exp_sign}{str(abs(power)).zfill(exponent.count('0'))}"+ text += f"E{exp_sign}{str(abs(power)).zfill(exp_digits)}"return prefix + text + percent + suffix-def _format_number(value: float, code: str) -> str | None:- """``value`` as a numeric format displays it, or None for formats left as stored."""+def _format_section(value: float, tokens: list[tuple[str, str]]) -> str | None:+ return _apply_section(value, _section_plan(tokens))+++def _number_plan(code: str):+ """Each section of a numeric format read once (with whether it has tokens), or None if unsupported."""tokens = _format_tokens(code) if code else Noneif not tokens:return None@@ -377,26 +419,65 @@ def _format_number(value: float, code: str) -> str | None:sections.append([])else:sections[-1].append(token)- if value < 0 and len(sections) > 1 and sections[1]:- return _format_section(-value, sections[1])- if value == 0 and len(sections) > 2 and sections[2]:- return _format_section(0.0, sections[2])- text = _format_section(abs(value), sections[0])+ return [(bool(section), _section_plan(section)) for section in sections]+++def _apply_number(value: float, plan) -> str | None:+ if plan is None:+ return None+ if value < 0 and len(plan) > 1 and plan[1][0]:+ return _apply_section(-value, plan[1][1])+ if value == 0 and len(plan) > 2 and plan[2][0]:+ return _apply_section(0.0, plan[2][1])+ text = _apply_section(abs(value), plan[0][1])return None if text is None else ("-" + text if value < 0 else text)-def _cell_number(value: float, format_id: int, code: str, date1904: bool) -> str:- """A numeric cell as its format displays it, falling back to the stored number."""- if len(code) > _MAX_FORMAT_CHARS: # tokenized per cell+def _format_number(value: float, code: str) -> str | None:+ """``value`` as a numeric format displays it, or None for formats left as stored."""+ return _apply_number(value, _number_plan(code))+++_NUMBER_ERRORS = (ArithmeticError, ValueError, decimal.InvalidOperation)+++def _cell_plan(format_id: int, code: str):+ if len(code) > _MAX_FORMAT_CHARS:code = ""kind = _format_kind(format_id, code)if kind:- return _serial_text(value, kind, date1904)+ return kind, Nonetry:- text = _format_number(value, code or _BUILTIN_NUMBER_FORMATS.get(format_id, ""))- except (ArithmeticError, ValueError, decimal.InvalidOperation):- text = None- return _number(value) if text is None else text+ return None, _number_plan(code or _BUILTIN_NUMBER_FORMATS.get(format_id, ""))+ except _NUMBER_ERRORS:+ return None, None+++class _CellNumbers:+ """Shows numeric cells as their formats display them, reading each distinct format once per workbook."""++ def __init__(self, date1904: bool):+ self.date1904 = date1904+ self._plans: dict[tuple[int, str], tuple] = {}++ def __call__(self, value: float, format_id: int, code: str) -> str:+ key = (format_id, code)+ plan = self._plans.get(key)+ if plan is None:+ plan = self._plans[key] = _cell_plan(format_id, code)+ kind, number_plan = plan+ if kind:+ return _serial_text(value, kind, self.date1904)+ try:+ text = _apply_number(value, number_plan)+ except _NUMBER_ERRORS:+ text = None+ return _number(value) if text is None else text+++def _cell_number(value: float, format_id: int, code: str, date1904: bool) -> str:+ """A numeric cell as its format displays it, falling back to the stored number."""+ return _CellNumbers(date1904)(value, format_id, code)def _column_index(ref: str) -> int:@@ -453,6 +534,7 @@ def xlsx(path: str) -> list[Section]:fmt = int(xf.get("numFmtId", "0") or 0)style_formats.append((fmt, custom.get(fmt, "")))+ cell_number = _CellNumbers(date1904)sections: list[Section] = []seen: set[int] = set()budget = _Budget()@@ -490,7 +572,7 @@ def xlsx(path: str) -> list[Section]:fmt, code = (style_formats[style] if style < len(style_formats) else (0, ""))- value = _cell_number(number, fmt, code, date1904)+ value = cell_number(number, fmt, code)cells.extend([""] * (col - len(cells)))if col == len(cells):cells.append(value)@@ -1377,9 +1459,11 @@ def xls(path: str) -> list[Section]:elif kind == 0x000A: # end of the workbook globalsbreak+ cell_number = _CellNumbers(date1904)+def number(value: float, xf: int) -> str:fmt = xf_formats[xf] if xf < len(xf_formats) else 0- return _cell_number(value, fmt, formats.get(fmt, ""), date1904)+ return cell_number(value, fmt, formats.get(fmt, ""))sections: list[Section] = []# BIFF cell records are a few bytes each, so 200 MB of them would hold tens of millions ofdiff --git a/studio/backend/tests/test_rag_office_formats.py b/studio/backend/tests/test_rag_office_formats.pyindex 2de02b25316..508b5bca8dc 100644--- a/studio/backend/tests/test_rag_office_formats.py+++ b/studio/backend/tests/test_rag_office_formats.py@@ -634,6 +634,56 @@ def test_xlsx_shows_numbers_as_formatted(tmp_path, serial, number_format, expectassert _text(path) == f"Sheet: T\n{expected}"+def test_xlsx_reads_each_number_format_once_per_workbook(tmp_path, monkeypatch):+ # Styles 1 and 3 share a format; style 2 is a built-in date with no code; style 4 a built-in number.+ styles = (+ f'<styleSheet {S}><numFmts><numFmt numFmtId="164" formatCode="#,##0.00;(#,##0.00);"zero""/>'+ '<numFmt numFmtId="165" formatCode="0.0%"/></numFmts><cellXfs><xf numFmtId="0"/>'+ '<xf numFmtId="164"/><xf numFmtId="14"/><xf numFmtId="164"/><xf numFmtId="3"/><xf numFmtId="165"/></cellXfs></styleSheet>'+ )+ money = '#,##0.00;(#,##0.00);"zero"'+ formats = {1: (164, money), 2: (14, ""), 3: (164, money), 4: (3, ""), 5: (165, "0.0%")}+ values = [(1234.5, 1), (-2.25, 3), (0, 1), (46000, 2), (1234567, 4), (0.125, 5), (-0.5, 5)]+ rows = "".join(+ f'<row r="{r}">'+ + "".join(+ f'<c r="{chr(65 + i)}{r}" s="{style}"><v>{value * r}</v></c>'+ for i, (value, style) in enumerate(values)+ )+ + "</row>"+ for r in range(1, 41)+ )+ path = _zip(+ tmp_path / "formats.xlsx",+ {+ "xl/workbook.xml": f'<workbook {S}><sheets><sheet name="A" sheetId="1" r:id="rId1"/><sheet name="B" sheetId="2" r:id="rId2"/></sheets></workbook>',+ "xl/_rels/workbook.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="worksheets/sheet1.xml"/>'+ '<Relationship Id="rId2" Target="worksheets/sheet2.xml"/></Relationships>',+ "xl/styles.xml": styles,+ "xl/worksheets/sheet1.xml": f"<worksheet {S}><sheetData>{rows}</sheetData></worksheet>",+ "xl/worksheets/sheet2.xml": f"<worksheet {S}><sheetData>{rows}</sheetData></worksheet>",+ },+ )+ tokenized = []+ real_tokens = office_formats._format_tokens+ monkeypatch.setattr(+ office_formats, "_format_tokens", lambda code: tokenized.append(code) or real_tokens(code)+ )+ text = _text(path)+ assert sorted(tokenized) == ["#,##0", money, "0.0%"]+ expected = "\n".join(+ " | ".join(+ office_formats._cell_number(float(value * r), *formats[style], False)+ for value, style in values+ )+ for r in range(1, 41)+ )+ assert text == f"Sheet: A\n{expected}\nSheet: B\n{expected}"+ assert (+ text.splitlines()[1] == "1,234.50 | (2.25) | zero | 2025-12-09 | 1,234,567 | 12.5% | -50.0%"+ )++def test_xlsx_finds_the_workbook_through_package_relationships(tmp_path):path = _zip(tmp_path / "moved.xlsx",