Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio RAG: read each spreadsheet number format once per workbook (#16339)

Daniel Han02372492c33e
diff --git a/studio/backend/core/rag/office_formats.py b/studio/backend/core/rag/office_formats.py
index 576aebad406..afd24a878d8 100644
--- a/studio/backend/core/rag/office_formats.py
+++ b/studio/backend/core/rag/office_formats.py
@@ -322,10 +322,12 @@ def _group(digits: str) -> str:
return ",".join([digits[:head]] + [digits[i : i + 3] for i in range(head, len(digits), 3)])
-def _format_section(value: float, tokens: list[tuple[str, str]]) -> str | None:
- marks = [i for i, (kind, _) in enumerate(tokens) if kind in ("ph", "exp")]
+def _section_plan(tokens: list[tuple[str, str]]):
+ """The value-independent reading of one format section: its literal text when it shows no
+ digits, None when unsupported, else the layout ``_apply_section`` fills in."""
if not any(kind == "ph" and ch in "0#?" for kind, ch in tokens):
return "".join(text for kind, text in tokens if kind == "lit")
+ marks = [i for i, (kind, _) in enumerate(tokens) if kind in ("ph", "exp")]
first, last = marks[0], marks[-1]
if any(kind == "lit" for kind, _ in tokens[first:last]):
return None
@@ -333,24 +335,60 @@ def _format_section(value: float, tokens: list[tuple[str, str]]) -> str | None:
suffix = "".join(text for kind, text in tokens[last + 1 :] if kind == "lit")
pattern = "".join(ch if kind == "ph" else "E" + ch for kind, ch in tokens[first : last + 1])
percent = "%" * pattern.count("%")
- value *= 100 ** len(percent)
pattern = pattern.replace("%", "")
mantissa, _, exponent = pattern.partition("E")
sign_style, exponent = (exponent[:1], exponent[1:]) if exponent else ("", "")
whole, dot, fraction = mantissa.partition(".")
+ thousands = 0
while whole.endswith(","): # trailing commas scale by thousands
- whole, value = whole[:-1], value / 1000
+ whole, thousands = whole[:-1], thousands + 1
grouped, whole = "," in whole, whole.replace(",", "")
places, required = sum(fraction.count(c) for c in "0#?"), fraction.count("0")
- min_whole = whole.count("0")
+ return (
+ prefix,
+ suffix,
+ percent,
+ thousands,
+ grouped,
+ dot,
+ places,
+ required,
+ whole.count("0"),
+ sign_style,
+ max(len(whole), 1),
+ "#" in whole,
+ exponent.count("0"),
+ )
+
+
+def _apply_section(value: float, plan) -> str | None:
+ if plan is None or isinstance(plan, str):
+ return plan
+ (
+ prefix,
+ suffix,
+ percent,
+ thousands,
+ grouped,
+ dot,
+ places,
+ required,
+ min_whole,
+ sign_style,
+ width,
+ hashed,
+ exp_digits,
+ ) = plan
+ value *= 100 ** len(percent)
+ for _ in range(thousands):
+ value = value / 1000
power = 0
if sign_style:
- width = max(len(whole), 1)
if value:
power = math.floor(math.log10(value))
- power = power - power % width if "#" in whole else power - (max(min_whole, 1) - 1)
+ power = power - power % width if hashed else power - (max(min_whole, 1) - 1)
if float(_round(value / 10**power, places)) >= 10 ** max(width, min_whole, 1):
- power += width if "#" in whole else 1
+ power += width if hashed else 1
value /= 10**power
digits = _round(value, places)
int_part, _, frac_part = digits.partition(".")
@@ -362,12 +400,16 @@ def _format_section(value: float, tokens: list[tuple[str, str]]) -> str | None:
text = int_part + (dot + frac_part if dot else "")
if sign_style:
exp_sign = "-" if power < 0 else ("+" if sign_style == "+" else "")
- text += f"E{exp_sign}{str(abs(power)).zfill(exponent.count('0'))}"
+ text += f"E{exp_sign}{str(abs(power)).zfill(exp_digits)}"
return prefix + text + percent + suffix
-def _format_number(value: float, code: str) -> str | None:
- """``value`` as a numeric format displays it, or None for formats left as stored."""
+def _format_section(value: float, tokens: list[tuple[str, str]]) -> str | None:
+ return _apply_section(value, _section_plan(tokens))
+
+
+def _number_plan(code: str):
+ """Each section of a numeric format read once (with whether it has tokens), or None if unsupported."""
tokens = _format_tokens(code) if code else None
if not tokens:
return None
@@ -377,26 +419,65 @@ def _format_number(value: float, code: str) -> str | None:
sections.append([])
else:
sections[-1].append(token)
- if value < 0 and len(sections) > 1 and sections[1]:
- return _format_section(-value, sections[1])
- if value == 0 and len(sections) > 2 and sections[2]:
- return _format_section(0.0, sections[2])
- text = _format_section(abs(value), sections[0])
+ return [(bool(section), _section_plan(section)) for section in sections]
+
+
+def _apply_number(value: float, plan) -> str | None:
+ if plan is None:
+ return None
+ if value < 0 and len(plan) > 1 and plan[1][0]:
+ return _apply_section(-value, plan[1][1])
+ if value == 0 and len(plan) > 2 and plan[2][0]:
+ return _apply_section(0.0, plan[2][1])
+ text = _apply_section(abs(value), plan[0][1])
return None if text is None else ("-" + text if value < 0 else text)
-def _cell_number(value: float, format_id: int, code: str, date1904: bool) -> str:
- """A numeric cell as its format displays it, falling back to the stored number."""
- if len(code) > _MAX_FORMAT_CHARS: # tokenized per cell
+def _format_number(value: float, code: str) -> str | None:
+ """``value`` as a numeric format displays it, or None for formats left as stored."""
+ return _apply_number(value, _number_plan(code))
+
+
+_NUMBER_ERRORS = (ArithmeticError, ValueError, decimal.InvalidOperation)
+
+
+def _cell_plan(format_id: int, code: str):
+ if len(code) > _MAX_FORMAT_CHARS:
code = ""
kind = _format_kind(format_id, code)
if kind:
- return _serial_text(value, kind, date1904)
+ return kind, None
try:
- text = _format_number(value, code or _BUILTIN_NUMBER_FORMATS.get(format_id, ""))
- except (ArithmeticError, ValueError, decimal.InvalidOperation):
- text = None
- return _number(value) if text is None else text
+ return None, _number_plan(code or _BUILTIN_NUMBER_FORMATS.get(format_id, ""))
+ except _NUMBER_ERRORS:
+ return None, None
+
+
+class _CellNumbers:
+ """Shows numeric cells as their formats display them, reading each distinct format once per workbook."""
+
+ def __init__(self, date1904: bool):
+ self.date1904 = date1904
+ self._plans: dict[tuple[int, str], tuple] = {}
+
+ def __call__(self, value: float, format_id: int, code: str) -> str:
+ key = (format_id, code)
+ plan = self._plans.get(key)
+ if plan is None:
+ plan = self._plans[key] = _cell_plan(format_id, code)
+ kind, number_plan = plan
+ if kind:
+ return _serial_text(value, kind, self.date1904)
+ try:
+ text = _apply_number(value, number_plan)
+ except _NUMBER_ERRORS:
+ text = None
+ return _number(value) if text is None else text
+
+
+def _cell_number(value: float, format_id: int, code: str, date1904: bool) -> str:
+ """A numeric cell as its format displays it, falling back to the stored number."""
+ return _CellNumbers(date1904)(value, format_id, code)
def _column_index(ref: str) -> int:
@@ -453,6 +534,7 @@ def xlsx(path: str) -> list[Section]:
fmt = int(xf.get("numFmtId", "0") or 0)
style_formats.append((fmt, custom.get(fmt, "")))
+ cell_number = _CellNumbers(date1904)
sections: list[Section] = []
seen: set[int] = set()
budget = _Budget()
@@ -490,7 +572,7 @@ def xlsx(path: str) -> list[Section]:
fmt, code = (
style_formats[style] if style < len(style_formats) else (0, "")
)
- value = _cell_number(number, fmt, code, date1904)
+ value = cell_number(number, fmt, code)
cells.extend([""] * (col - len(cells)))
if col == len(cells):
cells.append(value)
@@ -1377,9 +1459,11 @@ def xls(path: str) -> list[Section]:
elif kind == 0x000A: # end of the workbook globals
break
+ cell_number = _CellNumbers(date1904)
+
def number(value: float, xf: int) -> str:
fmt = xf_formats[xf] if xf < len(xf_formats) else 0
- return _cell_number(value, fmt, formats.get(fmt, ""), date1904)
+ return cell_number(value, fmt, formats.get(fmt, ""))
sections: list[Section] = []
# BIFF cell records are a few bytes each, so 200 MB of them would hold tens of millions of
diff --git a/studio/backend/tests/test_rag_office_formats.py b/studio/backend/tests/test_rag_office_formats.py
index 2de02b25316..508b5bca8dc 100644
--- a/studio/backend/tests/test_rag_office_formats.py
+++ b/studio/backend/tests/test_rag_office_formats.py
@@ -634,6 +634,56 @@ def test_xlsx_shows_numbers_as_formatted(tmp_path, serial, number_format, expect
assert _text(path) == f"Sheet: T\n{expected}"
+def test_xlsx_reads_each_number_format_once_per_workbook(tmp_path, monkeypatch):
+ # Styles 1 and 3 share a format; style 2 is a built-in date with no code; style 4 a built-in number.
+ styles = (
+ f'<styleSheet {S}><numFmts><numFmt numFmtId="164" formatCode="#,##0.00;(#,##0.00);&quot;zero&quot;"/>'
+ '<numFmt numFmtId="165" formatCode="0.0%"/></numFmts><cellXfs><xf numFmtId="0"/>'
+ '<xf numFmtId="164"/><xf numFmtId="14"/><xf numFmtId="164"/><xf numFmtId="3"/><xf numFmtId="165"/></cellXfs></styleSheet>'
+ )
+ money = '#,##0.00;(#,##0.00);"zero"'
+ formats = {1: (164, money), 2: (14, ""), 3: (164, money), 4: (3, ""), 5: (165, "0.0%")}
+ values = [(1234.5, 1), (-2.25, 3), (0, 1), (46000, 2), (1234567, 4), (0.125, 5), (-0.5, 5)]
+ rows = "".join(
+ f'<row r="{r}">'
+ + "".join(
+ f'<c r="{chr(65 + i)}{r}" s="{style}"><v>{value * r}</v></c>'
+ for i, (value, style) in enumerate(values)
+ )
+ + "</row>"
+ for r in range(1, 41)
+ )
+ path = _zip(
+ tmp_path / "formats.xlsx",
+ {
+ "xl/workbook.xml": f'<workbook {S}><sheets><sheet name="A" sheetId="1" r:id="rId1"/><sheet name="B" sheetId="2" r:id="rId2"/></sheets></workbook>',
+ "xl/_rels/workbook.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="worksheets/sheet1.xml"/>'
+ '<Relationship Id="rId2" Target="worksheets/sheet2.xml"/></Relationships>',
+ "xl/styles.xml": styles,
+ "xl/worksheets/sheet1.xml": f"<worksheet {S}><sheetData>{rows}</sheetData></worksheet>",
+ "xl/worksheets/sheet2.xml": f"<worksheet {S}><sheetData>{rows}</sheetData></worksheet>",
+ },
+ )
+ tokenized = []
+ real_tokens = office_formats._format_tokens
+ monkeypatch.setattr(
+ office_formats, "_format_tokens", lambda code: tokenized.append(code) or real_tokens(code)
+ )
+ text = _text(path)
+ assert sorted(tokenized) == ["#,##0", money, "0.0%"]
+ expected = "\n".join(
+ " | ".join(
+ office_formats._cell_number(float(value * r), *formats[style], False)
+ for value, style in values
+ )
+ for r in range(1, 41)
+ )
+ assert text == f"Sheet: A\n{expected}\nSheet: B\n{expected}"
+ assert (
+ text.splitlines()[1] == "1,234.50 | (2.25) | zero | 2025-12-09 | 1,234,567 | 12.5% | -50.0%"
+ )
+
+
def test_xlsx_finds_the_workbook_through_package_relationships(tmp_path):
path = _zip(
tmp_path / "moved.xlsx",