Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(spreadsheet): stop inventing digits for .xls cells above 2**53 (#3700)
sxh1a7c5bbf52ae
diff --git a/headroom/transforms/spreadsheet_ingest.py b/headroom/transforms/spreadsheet_ingest.pyindex 0a185a361..ca77b0c4d 100644--- a/headroom/transforms/spreadsheet_ingest.py+++ b/headroom/transforms/spreadsheet_ingest.py@@ -94,7 +94,15 @@ def _xls_cell(cell: object, datemode: int) -> object:return datetime(year, month, day, hour, minute, second)if kind == xlrd.XL_CELL_BOOLEAN:return bool(value)- if kind == xlrd.XL_CELL_NUMBER and float(value).is_integer():+ # int() is only lossless while the double can represent consecutive integers.+ # Above the range it renders the double's exact value instead of the number the+ # sheet held -- 123456789012345678 is stored as 1.2345678901234568e+17 and+ # would print a fabricated ...680 to an agent that has no way to tell it is an+ # approximation. Fall through to the float, which is what the .xlsx loader+ # renders (#3695). The bound is inclusive: binary64 represents 2**53 and+ # -2**53 exactly and openpyxl loads those as integers, so excluding them put+ # the two loaders out of step at the boundary itself.+ if kind == xlrd.XL_CELL_NUMBER and float(value).is_integer() and abs(value) <= 2**53:return int(value)if kind == xlrd.XL_CELL_ERROR:# openpyxl with data_only=True gives the text Excel shows, e.g. #DIV/0!diff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.pyindex fab7ca15d..496e47782 100644--- a/tests/test_transforms_tabular.py+++ b/tests/test_transforms_tabular.py@@ -474,6 +474,109 @@ def test_load_xls_renders_a_time_only_cell_as_a_time(tmp_path) -> None:assert load_spreadsheet(xls_path)["Data"].splitlines()[1] == "12:00:00"+def test_load_xls_and_xlsx_agree_above_the_exact_integer_range(tmp_path) -> None:+ """A double cannot hold consecutive integers past 2**53.++ ``_xls_cell`` converted any integral double with ``int()``, so a sheet+ holding 123456789012345678 rendered the double's exact value, 123456789012345680+ -- two fabricated digits presented to an agent as a precise identifier. The+ .xlsx loader has always rendered the float, which at least says+ "approximate", so bounding the conversion to the exactly-representable range+ keeps the ``12.0 -> 12`` fix from #3616 and restores agreement (#3695).+ """+ xlwt = pytest.importorskip("xlwt")+ pytest.importorskip("xlrd")+ openpyxl = pytest.importorskip("openpyxl")++ from headroom.transforms.spreadsheet_ingest import load_spreadsheet++ small, big = 12, 1.2345678901234568e17++ xls_book = xlwt.Workbook()+ xls_sheet = xls_book.add_sheet("Data")+ xls_sheet.write(0, 0, "Small")+ xls_sheet.write(0, 1, "Big")+ xls_sheet.write(1, 0, small)+ xls_sheet.write(1, 1, big)+ xls_path = tmp_path / "legacy.xls"+ xls_book.save(xls_path)++ # openpyxl is the reference the .xls path is written against, so the expected+ # rendering is the float repr it yields for the same value.+ openpyxl_wb = openpyxl.Workbook()+ openpyxl_sheet = openpyxl_wb.active+ openpyxl_sheet.title = "Data"+ openpyxl_sheet.append(["Small", "Big"])+ openpyxl_sheet.append([small, big])+ openpyxl_wb.save(tmp_path / "modern.xlsx")++ xls_row = load_spreadsheet(xls_path)["Data"].splitlines()[1]+ xlsx_row = load_spreadsheet(tmp_path / "modern.xlsx")["Data"].splitlines()[1]+ small_field, big_field = xls_row.split(",")++ # The #3616 win has to survive the bound: a small whole number is still an int.+ assert small_field == "12"+ # And the fabricated integer must be gone: the cell is rendered as the double+ # it is, which reads as an approximation instead of an exact identifier.+ assert big_field == repr(big)+ assert big_field != str(int(big))+ # Parity, asserted against the other loader rather than against my own+ # expectation. Small values agree verbatim; above the range openpyxl writes a+ # double with only 15 significant digits, so that side loses a digit on its+ # own and the rows cannot be string-equal. The promise this fix makes is+ # about magnitude: the two loaders agree to well within one unit in the last+ # place of the stored value (16 here), and neither hands the agent the+ # exact-looking decimal of the typed number.+ xlsx_small, xlsx_big = xlsx_row.split(",")+ assert small_field == xlsx_small == "12"+ assert abs(float(big_field) - float(xlsx_big)) <= 16+++def test_load_xls_and_xlsx_agree_at_the_exact_integer_boundary(tmp_path) -> None:+ """2**53 and -2**53 are exactly representable and openpyxl loads them as+ integers, so the .xls path must convert them too - the bound is inclusive.+ One step outside, the double cannot hold the value; what matters is that no+ digits are invented, and the two loaders then differ only in the trailing+ ``.0`` that marks a value as approximate.+ """+ xlwt = pytest.importorskip("xlwt")+ pytest.importorskip("xlrd")+ openpyxl = pytest.importorskip("openpyxl")++ from headroom.transforms.spreadsheet_ingest import load_spreadsheet++ boundary = 2**53+ values = [boundary, -boundary, boundary - 2, boundary + 2]++ xls_book = xlwt.Workbook()+ xls_sheet = xls_book.add_sheet("Data")+ xls_sheet.write(0, 0, "Value")+ for row, value in enumerate(values, start=1):+ xls_sheet.write(row, 0, float(value))+ xls_path = tmp_path / "boundary.xls"+ xls_book.save(xls_path)++ xlsx_wb = openpyxl.Workbook()+ xlsx_sheet = xlsx_wb.active+ xlsx_sheet.title = "Data"+ xlsx_sheet.append(["Value"])+ for value in values:+ xlsx_sheet.append([int(value)])+ xlsx_path = tmp_path / "boundary.xlsx"+ xlsx_wb.save(xlsx_path)++ xls = [line.split(",")[0] for line in load_spreadsheet(xls_path)["Data"].splitlines()[1:]]+ xlsx = [line.split(",")[0] for line in load_spreadsheet(xlsx_path)["Data"].splitlines()[1:]]++ # Inside the range (and exactly on it) the two loaders agree verbatim.+ assert xls[0] == xlsx[0] == "9007199254740992"+ assert xls[1] == xlsx[1] == "-9007199254740992"+ assert xls[2] == xlsx[2] == "9007199254740990"+ # Above it the .xls side keeps the float marker, and the digits are the same.+ assert xls[3].removesuffix(".0") == xlsx[3] == "9007199254740994"+ assert all(not field.endswith(".0") or float(field) == int(float(field)) for field in xls)++def test_load_xls_and_xlsx_agree_on_the_same_values(tmp_path) -> None:"""The reference: openpyxl is what the .xls path is matching."""xlwt = pytest.importorskip("xlwt")