Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docreader): Excel 百分比被读小 100 倍 —— 加开关(默认关) (#4423)

linus-liu-weba9290dcb60e9
diff --git a/.env.example b/.env.example
index fb0529a9c..207586b7a 100644
--- a/.env.example
+++ b/.env.example
@@ -425,6 +425,17 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434
# DOCREADER_DOCX_MAX_PAGES=0
# MarkItDown(重型解析器)并发 worker 数。0 关闭限流;默认 1 适合单机/GPU 受限。
# DOCREADER_MARKITDOWN_MAX_WORKERS=1
+# ----- XLSX / Excel -----
+# 按显示格式物化百分比单元格。Excel 的百分比是显示格式而非存储值:单元格实存 0.6112,
+# 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的 61.12% 变成 0.6112
+# (100 倍偏差)。开启后百分比单元格变成字面量文本 "61.12%"。默认 false:开启会使单元格
+# 由数值变为字符串,改变下游排序/计算看到的内容;关闭时读取行为与升级前完全一致。
+# 单次上传可用 parser_engine_overrides.xlsx_percent_as_text 覆盖(true/false)。
+# DOCREADER_XLSX_PERCENT_AS_TEXT=false
+# 兜底:同一行有 >= N 个显式百分比格式单元格时,把该行漏标格式的 0<|v|<1 数字也按百分比
+# 处理(源文件偶有漏刷格式的格子,每命中一次打 INFO 日志)。仅在上一开关开启时生效;
+# 0=关闭该兜底(默认)。
+# DOCREADER_XLSX_PCT_ROW_HINT_MIN=0
# 出站 HTTP/HTTPS 代理(抓取网页 / 远程图片时用)。
# DOCREADER_EXTERNAL_HTTP_PROXY=
# DOCREADER_EXTERNAL_HTTPS_PROXY=
diff --git a/docker-compose.yml b/docker-compose.yml
index d79b93d13..1d9c544ca 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -463,6 +463,16 @@ services:
# ========== 解析通用 ==========
- DOCREADER_DOCX_MAX_PAGES=${DOCREADER_DOCX_MAX_PAGES:-0}
- DOCREADER_MARKITDOWN_MAX_WORKERS=${DOCREADER_MARKITDOWN_MAX_WORKERS:-1}
+ # ========== XLSX / Excel ==========
+ # 按显示格式物化百分比单元格:Excel 的百分比是显示格式而非存储值,单元格实存
+ # 0.6112 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的
+ # 61.12% 会变成 0.6112(100 倍偏差)。开启后该单元格变成字面量文本 "61.12%"。
+ # 默认 false:开启会使单元格由数值变为字符串,改变下游排序/计算看到的内容,
+ # 存量部署行为保持不变。单次上传可用 parser_engine_overrides.xlsx_percent_as_text 覆盖。
+ - DOCREADER_XLSX_PERCENT_AS_TEXT=${DOCREADER_XLSX_PERCENT_AS_TEXT:-false}
+ # 同一行有 >= N 个显式百分比格式单元格时,把该行漏标格式的 0<|v|<1 数字也按百分比
+ # 处理(源文件偶有漏刷格式的格子)。仅在上一开关开启时生效;0=关闭(默认)。
+ - DOCREADER_XLSX_PCT_ROW_HINT_MIN=${DOCREADER_XLSX_PCT_ROW_HINT_MIN:-0}
# 出站 HTTP/HTTPS 代理(抓取网页 / 远程图片时)
- DOCREADER_EXTERNAL_HTTP_PROXY=${DOCREADER_EXTERNAL_HTTP_PROXY:-}
- DOCREADER_EXTERNAL_HTTPS_PROXY=${DOCREADER_EXTERNAL_HTTPS_PROXY:-}
diff --git a/docreader/parser/excel_parser.py b/docreader/parser/excel_parser.py
index 163ed6ed8..645cf7f29 100644
--- a/docreader/parser/excel_parser.py
+++ b/docreader/parser/excel_parser.py
@@ -74,11 +74,16 @@ class ExcelParser(BaseParser):
file_name: str = "",
file_type: str | None = None,
xlsx_first_row_as_header: Any = False,
+ xlsx_percent_as_text: Any = None,
**kwargs: Any,
):
super().__init__(file_name=file_name, file_type=file_type, **kwargs)
self.xlsx_first_row_as_header = _parse_bool(xlsx_first_row_as_header)
-
+ # Tri-state: None (or an empty value) keeps the deployment default from
+ # DOCREADER_XLSX_PERCENT_AS_TEXT, an explicit value wins. Same
+ # precedence as the pdf_force_scanned parser override.
+ self.xlsx_percent_as_text = _parse_optional_bool(xlsx_percent_as_text)
+
def parse_into_text(self, content: bytes) -> Document:
"""Parse Excel file bytes into a Document object.
@@ -100,7 +105,11 @@ class ExcelParser(BaseParser):
source_blocks: List[dict] = []
start, end = 0, 0
- excel_file = _open_excel_file(content, file_type=self.file_type)
+ excel_file = _open_excel_file(
+ content,
+ file_type=self.file_type,
+ percent_as_text=self.xlsx_percent_as_text,
+ )
# Process each sheet in the Excel file
for excel_sheet_name in excel_file.sheet_names:
@@ -205,7 +214,19 @@ def _parse_bool(value: Any) -> bool:
return str(value).strip().lower() in {"1", "true", "yes", "on"}
-def _prepare_xlsx_bytes(data: bytes) -> bytes:
+def _parse_optional_bool(value: Any) -> bool | None:
+ """Parse a tri-state parser override; None keeps the deployment default."""
+ if value is None:
+ return None
+ if isinstance(value, bool):
+ return value
+ text = str(value).strip().lower()
+ if not text:
+ return None
+ return text in {"1", "true", "yes", "y", "on"}
+
+
+def _prepare_xlsx_bytes(data: bytes, percent_as_text: bool | None = None) -> bytes:
repaired = repair_xlsx_bytes(data)
if repaired is not None:
data = repaired
@@ -213,7 +234,7 @@ def _prepare_xlsx_bytes(data: bytes) -> bytes:
if readable is not None:
data = readable
try:
- return fill_merged_cells_xlsx(data)
+ return fill_merged_cells_xlsx(data, percent_as_text=percent_as_text)
except TypeError:
# A non-conforming styles.xml (empty/malformed fills) makes openpyxl's
# load_workbook raise before pandas ever sees the file (#3637).
@@ -222,10 +243,14 @@ def _prepare_xlsx_bytes(data: bytes) -> bytes:
sanitized = sanitize_xlsx_styles(data)
if sanitized is None:
raise
- return fill_merged_cells_xlsx(sanitized)
+ return fill_merged_cells_xlsx(sanitized, percent_as_text=percent_as_text)
-def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFile:
+def _open_excel_file(
+ content: bytes,
+ file_type: str | None = None,
+ percent_as_text: bool | None = None,
+) -> pd.ExcelFile:
"""Open an Excel workbook with explicit engine selection and fallbacks."""
data = content
converted_via_soffice = False
@@ -254,7 +279,7 @@ def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFi
engine = engine_for_format(ext)
if ext == "xlsx":
- data = _prepare_xlsx_bytes(data)
+ data = _prepare_xlsx_bytes(data, percent_as_text=percent_as_text)
engine = "openpyxl"
try:
return pd.ExcelFile(BytesIO(data), engine=engine)
@@ -269,7 +294,7 @@ def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFi
if repaired is None:
raise
logger.info("Repaired XLSX sharedStrings packaging before parse")
- data = _prepare_xlsx_bytes(repaired)
+ data = _prepare_xlsx_bytes(repaired, percent_as_text=percent_as_text)
continue
except ValueError as exc:
if converted_via_soffice or "cannot be determined" not in str(exc):
diff --git a/docreader/parser/xlsx_merge.py b/docreader/parser/xlsx_merge.py
index a0613bdb8..12ccddebe 100644
--- a/docreader/parser/xlsx_merge.py
+++ b/docreader/parser/xlsx_merge.py
@@ -3,18 +3,185 @@
from __future__ import annotations
import logging
+import os
+import re
import zipfile
from io import BytesIO
logger = logging.getLogger(__name__)
+# --- Percentage number formats (opt-in) --------------------------------------
+# Excel keeps a percentage as the stored fraction plus a display format: the
+# cell holds 0.61123366 and "0.00%" renders it as 61.12%. openpyxl and pandas
+# read only the fraction, so a value the user sees as 61.12% reaches the
+# pipeline as 0.6112 — a 100x error that a model will copy verbatim into a
+# report. When this switch is on, percentage cells are materialized as the
+# literal text the spreadsheet shows ("61.12%") instead.
+#
+# It is off by default on purpose: the cell changes from a number into a
+# string, so anything that sorts, aggregates or computes with it sees different
+# input. With it off, every workbook is read exactly as before. A single upload
+# can override it through parser_engine_overrides.xlsx_percent_as_text.
+PERCENT_AS_TEXT_ENV = "DOCREADER_XLSX_PERCENT_AS_TEXT"
-def fill_merged_cells_xlsx(content: bytes) -> bytes:
+# Only percentages are materialized. Currency and thousands separators change
+# how a number is displayed, not what it means ("1,234,567" and 1234567 are the
+# same number), so rewriting them would churn cells for no semantic gain, and
+# date formats are already converted to datetimes by openpyxl.
+
+# --- Same-row hint fallback (opt-in, off by default) -------------------------
+# Workbooks occasionally leave one cell of a percentage row without a
+# percentage format (a real financial statement had 折旧率 0.0592 formatted as
+# "0.00_ ;[Red]\-0.00\ "). When a row carries at least this many explicitly
+# percentage-formatted cells, an unformatted 0 < |value| < 1 number in that row
+# is treated as a percentage too. Only consulted while percent materialization
+# is on; 0 disables the fallback.
+ROW_HINT_MIN_ENV = "DOCREADER_XLSX_PCT_ROW_HINT_MIN"
+_DEFAULT_ROW_HINT_MIN = 0
+
+# Only materialize a known numeric percentage pattern. Excel literals, conditions,
+# scaling and arbitrary decorations require a full format renderer; leave those
+# values untouched rather than guessing that every '%' means multiply by 100.
+_PERCENT_PATTERN = re.compile(
+ r"(?P<sign>-?)(?P<open>\()?(?P<group>#,##)?0"
+ r"(?:\.(?P<decimals>0+))?%(?P<close>\))?"
+)
+_FORMAT_COLOR = re.compile(
+ r"\[(?:Black|Blue|Cyan|Green|Magenta|Red|White|Yellow|Color[0-9]+)\]", re.I
+)
+
+
+def _env_bool(name: str, default: bool) -> bool:
+ value = os.environ.get(name)
+ if value is None or not str(value).strip():
+ return default
+ return str(value).strip().lower() in {"1", "true", "yes", "y", "on"}
+
+
+def _env_int(name: str, default: int) -> int:
+ try:
+ return int(str(os.environ.get(name, default)).strip())
+ except (TypeError, ValueError):
+ return default
+
+
+def percent_as_text_default() -> bool:
+ """Deployment-wide default for materializing percentage formats.
+
+ Read per call (not at import time) so the value can be flipped without a
+ module reload; a per-upload override still wins over it.
+ """
+ return _env_bool(PERCENT_AS_TEXT_ENV, False)
+
+
+def _format_percentage(value, number_format: str):
+ """Render supported Excel percentage formats, leaving other formats intact."""
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ return None
+ if not number_format or "%" not in number_format:
+ return None
+ # Quoted/escaped percentages are literal text, not percentage operators.
+ # Reject unsupported conditions across all sections before selecting one.
+ cleaned = _FORMAT_COLOR.sub("", number_format)
+ if any(char in cleaned for char in ('"', "\\", "_", "*", "[", "]")):
+ return None
+ sections = cleaned.split(";")
+ if len(sections) > 4:
+ return None
+ section_index = 1 if value < 0 and len(sections) >= 2 else 0
+ if value == 0 and len(sections) >= 3:
+ section_index = 2
+ match = _PERCENT_PATTERN.fullmatch(sections[section_index].strip())
+ if not match or bool(match["open"]) != bool(match["close"]):
+ return None
+ decimals = len(match["decimals"] or "")
+ # Excel's negative section formats the magnitude; a single section instead
+ # supplies the minus sign automatically.
+ amount = abs(value) if section_index == 1 else value
+ grouping = "," if match["group"] else ""
+ try:
+ number = f"{float(amount) * 100:{grouping}.{decimals}f}"
+ return f"{match['sign']}{match['open'] or ''}{number}%{match['close'] or ''}"
+ except (TypeError, ValueError, OverflowError):
+ return None
+
+
+def _materialize_percentage_cells(ws, row_hint_min: int) -> tuple[int, int]:
+ """Rewrite the percentage-formatted numbers of *ws* as their display text.
+
+ Must run before merged ranges are filled: only the master cell of a merge
+ carries a number_format while the covered cells are General, so filling
+ first would copy a bare fraction into rows 2..n and leave it unconverted —
+ the same row would then show both "0.55%" and 0.005509433890047721.
+
+ Returns ``(materialized, hinted)`` cell counts.
+ """
+ row_hints: dict = {}
+ if row_hint_min > 0:
+ # Count the explicit percentage formats before converting anything:
+ # the conversion below resets number_format to General.
+ for row in ws.iter_rows():
+ count = sum(
+ 1
+ for cell in row
+ if type(cell.value) in (int, float) # noqa: E721 - excludes bool
+ and _format_percentage(cell.value, cell.number_format) is not None
+ )
+ if count:
+ row_hints[row[0].row] = count
+
+ materialized = 0
+ hinted = 0
+ for row in ws.iter_rows():
+ hint = row_hints.get(row[0].row, 0)
+ for cell in row:
+ if type(cell.value) not in (int, float): # noqa: E721 - excludes bool
+ continue
+ text = _format_percentage(cell.value, cell.number_format)
+ if (
+ text is None
+ and "%" not in str(cell.number_format or "")
+ and hint >= row_hint_min > 0
+ and 0 < abs(cell.value) < 1
+ ):
+ # Fallback for a cell that lost its percentage format while its
+ # row kept one. Logged because it rewrites an unformatted value.
+ text = f"{float(cell.value) * 100:.2f}%"
+ hinted += 1
+ logger.info(
+ "XLSX pre-parse: %s!%s%d has no percentage format but its row "
+ "does; reading it as %.2f%% (stored value %r)",
+ ws.title,
+ cell.column_letter,
+ cell.row,
+ float(cell.value) * 100,
+ cell.value,
+ )
+ if text is None:
+ continue
+ cell.value = text
+ cell.number_format = "General"
+ materialized += 1
+ return materialized, hinted
+
+
+def fill_merged_cells_xlsx(
+ content: bytes, percent_as_text: bool | None = None
+) -> bytes:
"""Unmerge ranges and copy the master cell value into every covered cell.
openpyxl only stores values on the top-left cell of a merge; pandas then
sees NaN in the rest. Filling makes row-wise RAG chunks retain context.
+
+ ``percent_as_text`` also materializes percentage-formatted numbers as the
+ text the spreadsheet displays ("0.6112" -> "61.12%"), because the stored
+ fraction is 100x off from what a reader sees. ``None`` (the default) uses
+ the ``DOCREADER_XLSX_PERCENT_AS_TEXT`` deployment default, which is off, so
+ the default path is unchanged.
"""
+ if percent_as_text is None:
+ percent_as_text = percent_as_text_default()
+
if not zipfile.is_zipfile(BytesIO(content)):
return content
@@ -22,7 +189,21 @@ def fill_merged_cells_xlsx(content: bytes) -> bytes:
wb = load_workbook(BytesIO(content), data_only=True)
changed = False
+ materialized_cells = 0
+ hinted_cells = 0
+ row_hint_min = (
+ _env_int(ROW_HINT_MIN_ENV, _DEFAULT_ROW_HINT_MIN) if percent_as_text else 0
+ )
+
for ws in wb.worksheets:
+ if percent_as_text:
+ # Before the merge fill below — see _materialize_percentage_cells.
+ materialized, hinted = _materialize_percentage_cells(ws, row_hint_min)
+ materialized_cells += materialized
+ hinted_cells += hinted
+ if materialized:
+ changed = True
+
if not ws.merged_cells.ranges:
continue
for merge_range in list(ws.merged_cells.ranges):
@@ -52,5 +233,13 @@ def fill_merged_cells_xlsx(content: bytes) -> bytes:
out = BytesIO()
wb.save(out)
+ if materialized_cells:
+ logger.info(
+ "Materialized %d percentage cell(s) in XLSX before parse "
+ "(%d recovered by the same-row hint fallback, threshold=%d)",
+ materialized_cells,
+ hinted_cells,
+ row_hint_min,
+ )
logger.info("Filled merged cells in XLSX before parse")
return out.getvalue()
diff --git a/docreader/tests/test_xlsx_percent_format.py b/docreader/tests/test_xlsx_percent_format.py
new file mode 100644
index 000000000..5a8b81de3
--- /dev/null
+++ b/docreader/tests/test_xlsx_percent_format.py
@@ -0,0 +1,377 @@
+"""Tests for materializing XLSX percentage number formats (opt-in).
+
+Excel keeps a percentage as the underlying fraction plus a display format:
+the cell stores ``0.61123366`` and ``0.00%`` renders it as ``61.12%``.
+openpyxl and pandas read the fraction and drop the format, so a cell the user
+sees as ``61.12%`` reaches the pipeline as ``0.6112`` — a 100x error that a
+model will happily copy into a report.
+
+When ``DOCREADER_XLSX_PERCENT_AS_TEXT`` is on (or the per-upload
+``parser_engine_overrides`` key ``xlsx_percent_as_text`` is set), percentage
+cells are materialized as the literal text the user sees. The switch is off by
+default because the value changes from a number into a string, so anything that
+sorts or aggregates the cell would see different input; with it off every
+workbook is read exactly as before.
+"""
+
+import datetime
+import io
+import os
+import unittest
+from unittest.mock import patch
+
+import openpyxl
+
+from docreader.parser.excel_parser import ExcelParser
+from docreader.parser.parser import Parser
+from docreader.parser.xlsx_merge import fill_merged_cells_xlsx, _format_percentage
+
+PERCENT_ENV = "DOCREADER_XLSX_PERCENT_AS_TEXT"
+ROW_HINT_ENV = "DOCREADER_XLSX_PCT_ROW_HINT_MIN"
+OVERRIDE_KEY = "xlsx_percent_as_text"
+
+
+def _workbook_with_percentages() -> bytes:
+ """A real .xlsx mixing percentages, a merge, plain numbers and dates.
+
+ Layout (1-based; ``B3:B4`` is merged, so row 4's B cell is covered and has
+ no format of its own — exactly like the yearly columns of a financial
+ statement, where a label merge covers the row below):
+
+ A B C
+ 1 科目 2024 2025
+ 2 毛利率 0.61123366 (0.00%) 0.5512345 (0.00%)
+ 3 收入占比 0.0833 (0.00%) 0.0912 (0.00%)
+ 4 收入占比(续) <covered by B3:B4> 0.005509433890047721 (0.00%)
+ 5 营业收入 1234567.89 (#,##0.00) 9876543.21 (#,##0.00)
+ 6 报告日 2025-01-31 (yyyy-mm-dd) 2024-12-31 (yyyy-mm-dd)
+ 7 汇总 0.5 (General) 6100 (General)
+
+ Row 4 is the regression case: it carries its own percentage cell in C, so a
+ merge fill that runs before the percentage pass leaves that row showing both
+ ``0.55%`` and a raw ``0.0833``.
+ """
+ wb = openpyxl.Workbook()
+ ws = wb.active
+ ws.title = "财务"
+ ws.append(["科目", "2024", "2025"])
+ ws.append(["毛利率", 0.61123366, 0.5512345])
+ ws.append(["收入占比", 0.0833, 0.0912])
+ ws.append(["收入占比(续)", None, 0.005509433890047721])
+ ws.append(["营业收入", 1234567.89, 9876543.21])
+ ws.append(["报告日", datetime.date(2025, 1, 31), datetime.date(2024, 12, 31)])
+ ws.append(["汇总", 0.5, 6100])
+
+ for row in (2, 3):
+ ws.cell(row=row, column=2).number_format = "0.00%"
+ ws.cell(row=row, column=3).number_format = "0.00%"
+ ws["C4"].number_format = "0.00%"
+ # Merged after the formats are set: openpyxl keeps only the master (B3)
+ # style, which is what makes the covered cell's format General.
+ ws.merge_cells("B3:B4")
+ for row in (5,):
+ ws.cell(row=row, column=2).number_format = "#,##0.00"
+ ws.cell(row=row, column=3).number_format = "#,##0.00"
+ ws["B6"].number_format = "yyyy-mm-dd"
+ ws["C6"].number_format = "yyyy-mm-dd"
+
+ bio = io.BytesIO()
+ wb.save(bio)
+ return bio.getvalue()
+
+
+def _workbook_with_unformatted_cell_in_percentage_row() -> bytes:
+ """A percentage row (3 formatted cells) with one cell left General (D2)."""
+ wb = openpyxl.Workbook()
+ ws = wb.active
+ ws.append(["项目", "2024", "2025", "2026", "2027"])
+ ws.append(["折旧率", 0.0592, 0.0601, 0.5, 0.0612])
+ for col in (2, 3, 5):
+ ws.cell(row=2, column=col).number_format = "0.00%"
+ bio = io.BytesIO()
+ wb.save(bio)
+ return bio.getvalue()
+
+
+def _workbook_without_merges() -> bytes:
+ wb = openpyxl.Workbook()
+ ws = wb.active
+ ws.append(["项目", "值"])
+ ws.append(["占比", 0.25])
+ ws["B2"].number_format = "0%"
+ bio = io.BytesIO()
+ wb.save(bio)
+ return bio.getvalue()
+
+
+def _read_cells(content: bytes, *coordinates: str):
+ """Return the values of *coordinates* on the first worksheet."""
+ wb = openpyxl.load_workbook(io.BytesIO(content), data_only=True)
+ ws = wb.worksheets[0]
+ return [ws[c].value for c in coordinates]
+
+
+def _upstream_fill_merged_cells_xlsx(content: bytes) -> bytes:
+ """Frozen copy of the pre-change implementation, for byte-identity checks.
+
+ Kept verbatim (docreader/parser/xlsx_merge.py before the percentage switch)
+ so the default path of the new function can be compared byte for byte with
+ the behavior existing deployments have today.
+ """
+ import zipfile
+ from io import BytesIO
+
+ if not zipfile.is_zipfile(BytesIO(content)):
+ return content
+
+ from openpyxl import load_workbook
+
+ wb = load_workbook(BytesIO(content), data_only=True)
+ changed = False
+ for ws in wb.worksheets:
+ if not ws.merged_cells.ranges:
+ continue
+ for merge_range in list(ws.merged_cells.ranges):
+ master_value = ws.cell(merge_range.min_row, merge_range.min_col).value
+ ws.unmerge_cells(str(merge_range))
+ for row in range(merge_range.min_row, merge_range.max_row + 1):
+ for col in range(merge_range.min_col, merge_range.max_col + 1):
+ ws.cell(row, col).value = master_value
+ changed = True
+
+ if not changed:
+ return content
+
+ active_sheet = wb.active
+ for chart_sheet in list(wb.chartsheets):
+ wb.remove(chart_sheet)
+ if active_sheet in wb.worksheets:
+ wb.active = active_sheet
+ elif wb.worksheets:
+ wb.active = wb.worksheets[0]
+ for ws in wb.worksheets:
+ ws._charts.clear()
+
+ out = BytesIO()
+ wb.save(out)
+ return out.getvalue()
+
+
+class PercentMaterializationOffTests(unittest.TestCase):
+ """Default behavior: unchanged, and byte-identical to the old function."""
+
+ def test_default_keeps_the_raw_fraction(self):
+ content = _workbook_with_percentages()
+
+ self.assertEqual(
+ fill_merged_cells_xlsx(content), _upstream_fill_merged_cells_xlsx(content)
+ )
+
+ values = _read_cells(fill_merged_cells_xlsx(content), "B2", "C2", "B3", "B4")
+ self.assertEqual(values, [0.61123366, 0.5512345, 0.0833, 0.0833])
+
+ def test_excel_parser_default_keeps_the_raw_fraction(self):
+ document = ExcelParser().parse_into_text(_workbook_with_percentages())
+
+ self.assertIn("A: 毛利率,B: 0.61123366", document.content)
+ self.assertNotIn("61.12%", document.content)
+
+ def test_switch_off_is_byte_identical_to_the_upstream_implementation(self):
+ for build in (
+ _workbook_with_percentages,
+ _workbook_without_merges,
+ _workbook_with_unformatted_cell_in_percentage_row,
+ ):
+ content = build()
+ with self.subTest(fixture=build.__name__):
+ # Explicitly off, so a stray env var in the calling shell
+ # cannot turn this into an accidental behavior test.
+ with patch.dict(os.environ, {PERCENT_ENV: "0"}):
+ self.assertEqual(
+ fill_merged_cells_xlsx(content),
+ _upstream_fill_merged_cells_xlsx(content),
+ )
+
+ def test_env_var_off_forms_keep_the_raw_fraction(self):
+ content = _workbook_with_percentages()
+ for raw in ("0", "false", "FALSE", "no", "off", ""):
+ with self.subTest(value=raw):
+ with patch.dict(os.environ, {PERCENT_ENV: raw}):
+ self.assertEqual(
+ _read_cells(fill_merged_cells_xlsx(content), "B2"),
+ [0.61123366],
+ )
+
+
+class PercentMaterializationOnTests(unittest.TestCase):
+ """Enabled: percentages become the text the spreadsheet displays."""
+
+ def test_percent_cells_become_display_text(self):
+ content = _workbook_with_percentages()
+ with patch.dict(os.environ, {PERCENT_ENV: "true"}):
+ values = _read_cells(fill_merged_cells_xlsx(content), "B2", "C2", "B3", "C4")
+
+ self.assertEqual(values, ["61.12%", "55.12%", "8.33%", "0.55%"])
+
+ def test_merged_rows_below_the_master_are_materialized_too(self):
+ """The order pitfall: B4 is covered by B3:B4 and has no format itself."""
+ content = _workbook_with_percentages()
+ with patch.dict(os.environ, {PERCENT_ENV: "true"}):
+ b3, b4 = _read_cells(fill_merged_cells_xlsx(content), "B3", "B4")
+
+ self.assertEqual(b3, "8.33%")
+ self.assertEqual(b4, "8.33%")
+
+ def test_no_row_shows_both_a_percentage_and_a_raw_fraction(self):
+ with patch.dict(os.environ, {PERCENT_ENV: "true"}):
+ document = ExcelParser().parse_into_text(_workbook_with_percentages())
+
+ self.assertIn("A: 毛利率,B: 61.12%,C: 55.12%", document.content)
+ self.assertIn("A: 收入占比,B: 8.33%,C: 9.12%", document.content)
+ # Row 4 keeps its own percentage cell (C) and its merged B cell: both
+ # must be text, or the same row would read as two different scales.
+ self.assertIn("A: 收入占比(续),B: 8.33%,C: 0.55%", document.content)
+ for raw in ("0.61123366", "0.5512345", "0.0833", "0.0912", "0.005509433890047721"):
+ self.assertNotIn(raw, document.content)
+
+ def test_currency_plain_numbers_and_dates_are_not_touched(self):
+ content = _workbook_with_percentages()
+ with patch.dict(os.environ, {PERCENT_ENV: "true"}):
+ income, income_2025, plain, whole, start, end = _read_cells(
+ fill_merged_cells_xlsx(content), "B5", "C5", "B7", "C7", "B6", "C6"
+ )
+
+ # Thousands separators / currency only affect readability.
+ self.assertEqual(income, 1234567.89)
+ self.assertEqual(income_2025, 9876543.21)
+ # 0.5 has no percentage format, so it must stay a number.
+ self.assertEqual(plain, 0.5)
+ self.assertEqual(whole, 6100)
+ # openpyxl already turns date formats into datetimes.
+ self.assertEqual(start, datetime.datetime(2025, 1, 31))
+ self.assertEqual(end, datetime.datetime(2024, 12, 31))
+
+ def test_decimals_follow_the_number_format(self):
+ """The text mirrors what the format displays: 0%, 0.0%, 0.00%."""
+ wb = openpyxl.Workbook()
+ ws = wb.active
+ ws.append(["项目", "0%", "0.0%", "0.00%", "red"])
+ ws.append(["毛利率", 0.61123366, 0.61123366, 0.61123366, 0.61123366])
+ for col, fmt in (
+ (2, "0%"),
+ (3, "0.0%"),
+ (4, "#,##0.00%"),
+ (5, "0.00%;[Red]-0.00%"),
+ ):
+ ws.cell(row=2, column=col).number_format = fmt
+ bio = io.BytesIO()
+ wb.save(bio)
+
+ with patch.dict(os.environ, {PERCENT_ENV: "true"}):
+ values = _read_cells(
+ fill_merged_cells_xlsx(bio.getvalue()), "B2", "C2", "D2", "E2"
+ )
+
+ self.assertEqual(values, ["61%", "61.1%", "61.12%", "61.12%"])
+
+ def test_unformatted_cell_in_a_percentage_row_needs_the_opt_in_fallback(self):
+ content = _workbook_with_unformatted_cell_in_percentage_row()
+ with patch.dict(os.environ, {PERCENT_ENV: "true"}):
+ self.assertEqual(
+ _read_cells(fill_merged_cells_xlsx(content), "D2"), [0.5]
+ )
+ with patch.dict(
+ os.environ, {PERCENT_ENV: "true", ROW_HINT_ENV: "3"}
+ ):
+ self.assertEqual(
+ _read_cells(fill_merged_cells_xlsx(content), "D2"), ["50.00%"]
+ )
+
+
+class PercentageFormatSyntaxTests(unittest.TestCase):
+ def test_literal_and_conditional_percent_formats_are_not_scaled(self):
+ for fmt in (r"0.00\%", '0.00"%"', '[>1]0%;0.00%', '0.00%" growth"'):
+ with self.subTest(fmt=fmt):
+ self.assertIsNone(_format_percentage(0.5, fmt))
+
+ def test_selects_the_section_for_the_value(self):
+ for value, fmt, expected in (
+ (0.5, "0%;0.00%", "50%"),
+ (-0.5, "0%;[Red]-0.00%", "-50.00%"),
+ (-0.5, "0%;(0.00%)", "(50.00%)"),
+ (0, "0%;0.00%;0.0%", "0.0%"),
+ (-0.5, "0.00%", "-50.00%"),
+ (12.345, "#,##0.00%", "1,234.50%"),
+ ):
+ with self.subTest(value=value, fmt=fmt):
+ self.assertEqual(_format_percentage(value, fmt), expected)
+
+ def test_row_hint_does_not_convert_literal_percent_or_count_it_as_a_hint(self):
+ wb = openpyxl.Workbook()
+ ws = wb.active
+ ws.append(["label", "percent", "literal", "plain"])
+ ws.append(["mixed", 0.5, 0.5, 0.5])
+ ws.append(["literal only", 0.5, 0.5, 0.5])
+ ws["B2"].number_format = "0%"
+ ws["C2"].number_format = r"0.00\%"
+ ws["B3"].number_format = '0.00"%"'
+ data = io.BytesIO()
+ wb.save(data)
+ with patch.dict(os.environ, {PERCENT_ENV: "true", ROW_HINT_ENV: "1"}):
+ result = fill_merged_cells_xlsx(data.getvalue())
+ self.assertEqual(_read_cells(result, "B2", "C2", "D2"), ["50%", 0.5, "50.00%"])
+ self.assertEqual(_read_cells(result, "B3", "C3", "D3"), [0.5, 0.5, 0.5])
+
+
+class PercentMaterializationSwitchTests(unittest.TestCase):
+ """The switch itself: env default, per-upload override, precedence."""
+
+ def test_env_var_true_forms_enable_materialization(self):
+ content = _workbook_with_percentages()
+ for raw in ("1", "true", "TRUE", "yes", "y", "on"):
+ with self.subTest(value=raw):
+ with patch.dict(os.environ, {PERCENT_ENV: raw}):
+ self.assertEqual(
+ _read_cells(fill_merged_cells_xlsx(content), "B2"),
+ ["61.12%"],
+ )
+
+ def test_excel_parser_accepts_the_override_key(self):
+ document = ExcelParser(xlsx_percent_as_text="true").parse_into_text(
+ _workbook_with_percentages()
+ )
+
+ self.assertIn("A: 毛利率,B: 61.12%", document.content)
+
+ def test_parser_engine_overrides_key_enables_materialization(self):
+ document = Parser().parse_file(
+ "report.xlsx",
+ "xlsx",
+ _workbook_with_percentages(),
+ engine_overrides={OVERRIDE_KEY: "true"},
+ )
+
+ self.assertIn("A: 毛利率,B: 61.12%", document.content)
+
+ def test_upload_override_wins_over_the_env_default(self):
+ content = _workbook_with_percentages()
+
+ with patch.dict(os.environ, {PERCENT_ENV: "true"}):
+ off = ExcelParser(xlsx_percent_as_text="false").parse_into_text(content)
+ with patch.dict(os.environ, {PERCENT_ENV: "false"}):
+ on = ExcelParser(xlsx_percent_as_text="true").parse_into_text(content)
+
+ self.assertNotIn("61.12%", off.content)
+ self.assertIn("B: 0.61123366", off.content)
+ self.assertIn("61.12%", on.content)
+
+ def test_absent_override_keeps_the_env_default(self):
+ content = _workbook_with_percentages()
+ with patch.dict(os.environ, {PERCENT_ENV: "true"}):
+ document = ExcelParser().parse_into_text(content)
+
+ self.assertIn("A: 毛利率,B: 61.12%", document.content)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/website-docs/03-features/03-document-parsing.md b/website-docs/03-features/03-document-parsing.md
index e59fd7817..0cecf4b28 100644
--- a/website-docs/03-features/03-document-parsing.md
+++ b/website-docs/03-features/03-document-parsing.md
@@ -202,10 +202,16 @@ sequenceDiagram
三个辅助模块解决现实中的脏文件:
- **`excel_convert.py`**:魔数/`inspect_excel_format` 检测真实格式(xlsx/xls/xlsb/ods),为每种格式选 pandas engine(`xlrd`/`openpyxl`/`odf`);无法识别时(如 WPS `.et`、被改名的 csv)用 LibreOffice `convert-to xlsx` 归一化(`normalize_excel_bytes` 依次尝试 `.xlsx/.xls/.et/.csv` 后缀)。
-- **`xlsx_merge.py`**:`fill_merged_cells_xlsx` 解除合并单元格并把左上角主值**复制到覆盖区域的每个单元格**——openpyxl 只在左上角存值,pandas 会把其余读成 NaN,填充后按行分块的 RAG chunk 才能保留上下文。
+- **`xlsx_merge.py`**:`fill_merged_cells_xlsx` 解除合并单元格并把左上角主值**复制到覆盖区域的每个单元格**——openpyxl 只在左上角存值,pandas 会把其余读成 NaN,填充后按行分块的 RAG chunk 才能保留上下文。可选地**按显示格式物化百分比**(见下)。
- **`xlsx_repair.py`**:修复常见 XLSX 打包问题——`sharedStrings.xml` 路径大小写/位置不规范时重命名归位;manifest 引用了 sharedStrings 但包内缺失且工作表只用 inline string 时,从 `[Content_Types].xml` 与 workbook rels 中剥掉引用,使 openpyxl 能读。
-XLSX 读取前统一走 `repair → fill_merged_cells` 预处理,并用 `header=None` + A/B/C 列字母作为稳定列名(xls 则先尝试首行做表头,遇 `Unnamed:` 列回退列字母)。
+**百分比格式**:Excel 的百分比是**显示格式而非存储值**——单元格实存 `0.61123366`,靠 `0.00%` 渲染成 `61.12%`,而 pandas/openpyxl 读的是原始值,格式被丢掉,于是 `61.12%` 变成 `0.6112`(**100 倍偏差**,财务报表类数据会直接把错数字喂给模型)。打开 `DOCREADER_XLSX_PERCENT_AS_TEXT`(或单次上传给 `parser_engine_overrides.xlsx_percent_as_text=true`,优先级高于环境变量)后,带百分比格式的数值单元格会被就地转成显示文本 `"61.12%"`(格式置 `General`),转换在**填合并单元格之前**完成——只有合并区域左上角带 `number_format`,被覆盖的格子是 `General`,先填合并会让第 2..n 行漏转。**只处理百分比**:货币/千分位只影响可读性(`1,234,567` 和 `1234567` 是同一个数),日期由 openpyxl 自动转 datetime。默认 **false**——开启后该单元格由数值变成字符串,会影响下游排序/计算,关闭时读取行为与升级前完全一致。
+
+格式识别仅转换可可靠渲染的数字百分比模式(含颜色及正数/负数/零分段)。转义或引号中的 `%` 是字面量,不会触发乘 100;条件、填充和其他复杂自定义格式保留原始值,避免错误缩放。字面量百分号单元格也不会触发行提示推断。
+
+同源文件偶尔有「一格忘了刷百分比格式」的单元格(如某财报 `D12` 折旧率 `0.0592` 格式为 `0.00_ ;[Red]\-0.00\ `)。把 `DOCREADER_XLSX_PCT_ROW_HINT_MIN` 设为 N(默认 **0=关闭**)后,同一行有 ≥N 个显式百分比格式单元格时,该行漏标的 `0<|v|<1` 数字也按百分比处理,每次命中打 INFO 日志可事后审计。
+
+XLSX 读取前统一走 `repair → fill_merged_cells` 预处理,并用 `header=None` + A/B/C 列字母作为稳定列名(xls 则先尝试首行做表头,遇 `Unnamed:` 列回退列字母)。百分比物化(若开启)发生在 `fill_merged_cells` 内部、填合并单元格之前。
#### ppt_convert.py / pptx_media.py(.ppt / .pptx,服务于 markitdown 引擎) {#_3-5-ppt-convert-py-pptx-media-py-ppt-pptx-服务于-markitdown-引擎}