A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(table): preserve pandas sentinels by output mode (#10618)
Kiran Gadhavef470f31fc5eb
diff --git a/frontend/src/components/data-table/__tests__/columns.test.tsx b/frontend/src/components/data-table/__tests__/columns.test.tsxindex a2561ac1fb..b77d62f5d1 100644--- a/frontend/src/components/data-table/__tests__/columns.test.tsx+++ b/frontend/src/components/data-table/__tests__/columns.test.tsx@@ -788,13 +788,13 @@ describe("LocaleNumber", () => {});describe("renderCellValue with string + edge whitespace", () => {- const createMockStringColumn = () =>+ const createMockStringColumn = (dtype = "object") =>({id: "desc",columnDef: {meta: {dataType: "string" as const,- dtype: "object",+ dtype,},},getColumnFormatting: () => undefined,@@ -851,6 +851,21 @@ describe("renderCellValue with string + edge whitespace", () => {}});+ it("renders pandas timedelta NaT as a sentinel", () => {+ const mockColumn = createMockStringColumn("timedelta64[us]");+ const result = renderCellValue({+ column: mockColumn,+ renderValue: () => "NaT",+ getValue: () => "NaT",+ selectCell: undefined,+ cellStyles: "",+ });++ const { container } = renderWithProviders(result);++ expect(container.querySelector('[aria-label="Not a Time"]')).not.toBeNull();+ });+it("renders edge whitespace markers and still detects the URL in the middle", () => {const mockColumn = createMockStringColumn();const value = " https://example.com ";diff --git a/frontend/src/components/data-table/__tests__/filter-by-values-picker.test.tsx b/frontend/src/components/data-table/__tests__/filter-by-values-picker.test.tsxindex 639c388db4..25d7c91f01 100644--- a/frontend/src/components/data-table/__tests__/filter-by-values-picker.test.tsx+++ b/frontend/src/components/data-table/__tests__/filter-by-values-picker.test.tsx@@ -10,10 +10,10 @@ beforeAll(() => {};});-function mockColumn(): Column<unknown, unknown> {+function mockColumn(dtype?: string): Column<unknown, unknown> {return {id: "name",- columnDef: { meta: { dataType: "string" } },+ columnDef: { meta: { dataType: "string", dtype } },} as unknown as Column<unknown, unknown>;}@@ -27,6 +27,19 @@ async function calculateTopK() {}describe("FilterByValuesList — creatable", () => {+ it("renders pandas timedelta NaT as a sentinel", async () => {+ render(+ <FilterByValuesList+ column={mockColumn("timedelta64[us]")}+ calculateTopKRows={async () => ({ data: [["NaT", 1]] })}+ chosenValues={new Set()}+ onChange={vi.fn()}+ />,+ );++ expect(await screen.findByLabelText("Not a Time")).toBeInTheDocument();+ });+it("shows '+ Add \"X\"' item when creatable and query is non-empty", async () => {const onChange = vi.fn();render(diff --git a/frontend/src/components/data-table/__tests__/utils.test.ts b/frontend/src/components/data-table/__tests__/utils.test.tsindex 0fafcdf232..7dc08b2c04 100644--- a/frontend/src/components/data-table/__tests__/utils.test.ts+++ b/frontend/src/components/data-table/__tests__/utils.test.ts@@ -301,6 +301,8 @@ describe("detectSentinel", () => {it("should not match NaT in non-temporal columns", () => {expect(detectSentinel("NaT", "string")).toBeNull();+ expect(detectSentinel("NaT", "string", "object")).toBeNull();+ expect(detectSentinel("NaT", "string", "category")).toBeNull();});it("should match NaT in temporal columns", () => {@@ -313,6 +315,26 @@ describe("detectSentinel", () => {value: "NaT",});});++ it("should match NaT in pandas timedelta and period columns", () => {+ expect(detectSentinel("NaT", "string", "timedelta64[us]")).toEqual({+ type: "nat",+ value: "NaT",+ });+ expect(detectSentinel("NaT", "unknown", "period[M]")).toEqual({+ type: "nat",+ value: "NaT",+ });+ });++ it("should match NaT in Narwhals duration columns", () => {+ expect(detectSentinel("NaT", "number", "Duration(time_unit='us')")).toEqual(+ {+ type: "nat",+ value: "NaT",+ },+ );+ });});function createMockTableWithMeta<TData>(rawData?: TData[]): Table<TData> {diff --git a/frontend/src/components/data-table/columns.tsx b/frontend/src/components/data-table/columns.tsxindex 7818a5d001..1bd6793a44 100644--- a/frontend/src/components/data-table/columns.tsx+++ b/frontend/src/components/data-table/columns.tsx@@ -634,7 +634,7 @@ export function renderCellValue<TData, TValue>({// Sentinel values (null, whitespace, NaN, Infinity, NaT) rendered specially.// Empty strings are left as-is- const sentinel = detectSentinel(value, dataType);+ const sentinel = detectSentinel(value, dataType, dtype);if (sentinel && sentinel.type !== "empty-string") {return (<div onClick={selectCell} className={cellStyles}>diff --git a/frontend/src/components/data-table/filter-by-values-picker.tsx b/frontend/src/components/data-table/filter-by-values-picker.tsxindex 0b21a8bce0..1570a49a19 100644--- a/frontend/src/components/data-table/filter-by-values-picker.tsx+++ b/frontend/src/components/data-table/filter-by-values-picker.tsx@@ -278,6 +278,7 @@ export const FilterByValuesList = <TData, TValue>({const sentinel = detectSentinel(value,column.columnDef.meta?.dataType,+ column.columnDef.meta?.dtype,);return (<CommandItemdiff --git a/frontend/src/components/data-table/filters/types.ts b/frontend/src/components/data-table/filters/types.tsindex b8c958c8f6..0c6b862ed8 100644--- a/frontend/src/components/data-table/filters/types.ts+++ b/frontend/src/components/data-table/filters/types.ts@@ -7,8 +7,10 @@ declare module "@tanstack/react-table" {//allows us to define custom properties for our columnsinterface ColumnMeta<TData extends RowData, TValue> {rowHeader?: boolean;- dtype?: string;+ /** Normalized marimo type used for backend-independent UI behavior. */dataType?: DataType;+ /** Source-library type, such as pandas `timedelta64[ns]`. */+ dtype?: string;filterType?: FilterType;minFractionDigits?: number;width?: number;diff --git a/frontend/src/components/data-table/row-viewer-panel/__tests__/row-viewer.test.tsx b/frontend/src/components/data-table/row-viewer-panel/__tests__/row-viewer.test.tsxindex 401b42e474..9a5e2aa79c 100644--- a/frontend/src/components/data-table/row-viewer-panel/__tests__/row-viewer.test.tsx+++ b/frontend/src/components/data-table/row-viewer-panel/__tests__/row-viewer.test.tsx@@ -55,4 +55,20 @@ describe("RowExpandedPanel", () => {expect(await screen.findByText("John")).toBeInTheDocument();expect(await screen.findByText("30")).toBeInTheDocument();});++ it("renders pandas timedelta NaT as a sentinel", async () => {+ renderWithProviders(+ <RowViewerPanel+ rowIdx={0}+ setRowIdx={mockSetRowIdx}+ totalRows={1}+ fieldTypes={[["duration", ["string", "timedelta64[us]"]]]}+ getRow={async () => ({ rows: [{ duration: "NaT" }] })}+ isSelectable={false}+ isRowSelected={false}+ />,+ );++ expect(await screen.findByLabelText("Not a Time")).toBeInTheDocument();+ });});diff --git a/frontend/src/components/data-table/row-viewer-panel/row-viewer.tsx b/frontend/src/components/data-table/row-viewer-panel/row-viewer.tsxindex e701027638..f9f3864d33 100644--- a/frontend/src/components/data-table/row-viewer-panel/row-viewer.tsx+++ b/frontend/src/components/data-table/row-viewer-panel/row-viewer.tsx@@ -189,7 +189,7 @@ export const RowViewerPanel: React.FC<RowViewerPanelProps> = ({</TableRow></TableHeader><TableBody>- {fieldTypes?.map(([columnName, [dataType, _externalType]]) => {+ {fieldTypes?.map(([columnName, [dataType, externalType]]) => {const columnValue = rowValues[columnName];if (!inSearchQuery({ columnName, columnValue, searchQuery })) {@@ -201,6 +201,7 @@ export const RowViewerPanel: React.FC<RowViewerPanelProps> = ({columnDef: {meta: {dataType,+ dtype: externalType,},},getColumnFormatting: () => undefined,diff --git a/frontend/src/components/data-table/types.ts b/frontend/src/components/data-table/types.tsindex 13d14ec0c9..9741a57058 100644--- a/frontend/src/components/data-table/types.ts+++ b/frontend/src/components/data-table/types.ts@@ -53,7 +53,12 @@ export type ColumnHeaderStats = Record<export type FieldTypesWithExternalType = [columnName: string,- [dataType: DataType, externalType: string],+ [+ // Normalized marimo type, such as "string" or "datetime".+ dataType: DataType,+ // Source-library type, such as pandas "timedelta64[ns]".+ externalType: string,+ ],][];// Ordered map of column name -> data type.//diff --git a/frontend/src/components/data-table/utils.ts b/frontend/src/components/data-table/utils.tsindex 330902599d..d5e7e3f41a 100644--- a/frontend/src/components/data-table/utils.ts+++ b/frontend/src/components/data-table/utils.ts@@ -154,14 +154,39 @@ const NUMERIC_STRING_SPECIALS: Record<string, StringValueSentinelType> = {"-inf": "negative-infinity",};+// These source-library dtypes are temporal, but their normalized marimo data+// types are "string", "number", or "unknown" instead of a temporal type.+const NAT_DTYPE_PREFIXES = ["timedelta64[", "period[", "duration"] as const;++function isNaTType(+ dataType: DataType | undefined,+ dtype: string | undefined,+): boolean {+ if (isTemporalType(dataType)) {+ return true;+ }+ if (!dtype) {+ return false;+ }++ const normalizedDtype = dtype.toLowerCase();+ return NAT_DTYPE_PREFIXES.some((prefix) =>+ normalizedDtype.startsWith(prefix),+ );+}+/*** Detect if a cell value is a sentinel (null, empty string, whitespace,* NaN, infinity, NaT). Column-type-dependent sentinels (string "NaN",- * "NaT", etc.) are matched based on `dataType`.+ * "NaT", etc.) are matched based on `dataType` and `dtype`.+ *+ * `dataType` is marimo's normalized, backend-independent type. `dtype` is+ * the source library's more specific type string.*/export function detectSentinel(value: unknown,dataType: DataType | undefined,+ dtype?: string,): CellValueSentinel | null {if (value == null) {return { type: "null", value };@@ -181,8 +206,7 @@ export function detectSentinel(return { type, value };}}- // String "NaT" in a temporal column = pandas Not-a-Time sentinel- if (isTemporalType(dataType) && value === "NaT") {+ if (value === "NaT" && isNaTType(dataType, dtype)) {return { type: "nat", value };}return null;diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.pyindex 31246dc4a7..0b13786814 100644--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py@@ -278,12 +278,24 @@ class PandasTableManagerFactory(TableManagerFactory):strict_json: bool = False,ensure_ascii: bool = True,) -> str:+ """Serialize pandas rows for frontend display or strict export.++ The frontend path can emit `NaN`, `Infinity`, `-Infinity`,+ and typed `NaT` sentinels. The frontend parser and table+ renderer display these values.++ The strict path converts values that standard JSON does not+ support. Pandas encodes these values as JSON `null`.+ """+def to_json(result: pd.DataFrame,) -> list[dict[str, Any]] | str:- """- to_dict preserves nans, infs and is more accurate than to_json.- By default, we use to_dict unless strict_json is True+ """Serialize the prepared dataframe.++ The frontend path intentionally preserves non-finite+ sentinels through `to_dict`. The strict export path uses+ pandas JSON, which converts them to JSON `null`."""if strict_json:try:@@ -345,7 +357,13 @@ class PandasTableManagerFactory(TableManagerFactory):if is_timedelta64_dtype(dtype) or is_timedelta64_ns_dtype(dtype):- result[col] = _stringify_preserving_nulls(series)+ # Preserve the NaT sentinel for frontend rendering;+ # strict exports require JSON nulls.+ result[col] = (+ _stringify_preserving_nulls(series)+ if strict_json+ else series.apply(str)+ )if is_object_dtype(dtype):# Convert date objects to the YYYY-MM-DD display form.inferred_dtype = self._infer_dtype(col)diff --git a/marimo/_plugins/ui/_impl/tables/table_manager.py b/marimo/_plugins/ui/_impl/tables/table_manager.pyindex 3660537e5d..93a5e5fab8 100644--- a/marimo/_plugins/ui/_impl/tables/table_manager.py+++ b/marimo/_plugins/ui/_impl/tables/table_manager.py@@ -125,12 +125,20 @@ class TableManager(abc.ABC, Generic[T]):strict_json: bool = False,ensure_ascii: bool = True,) -> str:- pass+ """Serialize rows for frontend display or strict export.++ `strict_json=False` requests output for the frontend special-value+ parser. The output can contain bare non-finite numbers. A strict JSON+ parser can reject this output.++ `strict_json=True` requests standards-compliant output for downloads+ and external JSON consumers.+ """def to_json(self,format_mapping: FormatMapping | None = None,- strict_json: bool = False, # Whether the result should be strictly JSON compliant (eg. nan -> null)+ strict_json: bool = False,encoding: str | None = "utf-8",ensure_ascii: bool = True,) -> bytes:diff --git a/marimo/_smoke_tests/tables/pandas_null_stringify.py b/marimo/_smoke_tests/tables/pandas_null_stringify.pyindex 51a786809e..cbb30fad12 100644--- a/marimo/_smoke_tests/tables/pandas_null_stringify.py+++ b/marimo/_smoke_tests/tables/pandas_null_stringify.py@@ -40,7 +40,7 @@ def _(date, pd):@app.cell-def _(PandasTableManagerFactory, dataframe, json):+def _(PandasTableManagerFactory, dataframe, json, pd):manager = PandasTableManagerFactory.create()(dataframe)json_data = json.loads(manager.to_json_str())assert json_data == [@@ -53,12 +53,25 @@ def _(PandasTableManagerFactory, dataframe, json):},{"complex": None,- "timedelta": None,+ "timedelta": "NaT","date": None,"bytes": None,"extension": None,},]+ strict_json_data = json.loads(manager.to_json_str(strict_json=True))+ assert strict_json_data[1]["timedelta"] is None++ all_null_extension = pd.Series([None], dtype="category").to_frame(+ name="extension"+ )+ all_null_manager = PandasTableManagerFactory.create()(all_null_extension)+ assert all_null_manager.to_json_str(strict_json=False) == (+ '[{"extension":NaN}]'+ )+ assert all_null_manager.to_json_str(strict_json=True) == (+ '[{"extension":null}]'+ )json_datareturndiff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.pyindex dad7e7e902..aaccc09e16 100644--- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py+++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py@@ -11,6 +11,7 @@ from types import SimpleNamespacefrom typing import Anyfrom unittest.mock import Mock, patch+import msgspecimport narwhals.stable.v2 as nwimport pytest@@ -367,7 +368,6 @@ class TestPandasTableManager(unittest.TestCase):df = pd.DataFrame({"complex": [1 + 2j, complex(nan, nan)],- "timedelta": [pd.Timedelta(days=1), pd.NaT],"date": [datetime.date(2020, 1, 1), None],"bytes": [b"ab", None],}@@ -378,18 +378,32 @@ class TestPandasTableManager(unittest.TestCase):assert json_data == [{"complex": "(1+2j)",- "timedelta": "1 days 00:00:00","date": "2020-01-01","bytes": "b'ab'",},{"complex": None,- "timedelta": None,"date": None,"bytes": None,},]+ def test_to_json_str_preserves_timedelta_nat_for_frontend(self) -> None:+ df = pd.DataFrame({"value": [pd.Timedelta(days=1), pd.NaT]})+ manager = self.factory.create()(df)++ assert manager.to_json_str(strict_json=False) == (+ '[{"value":"1 days 00:00:00"},{"value":"NaT"}]'+ )++ def test_to_json_str_converts_timedelta_nat_for_export(self) -> None:+ df = pd.DataFrame({"value": [pd.Timedelta(days=1), pd.NaT]})+ manager = self.factory.create()(df)++ assert manager.to_json_str(strict_json=True) == (+ '[{"value":"1 days 00:00:00"},{"value":null}]'+ )+@pytest.mark.skipif(not DependencyManager.numpy.has(),reason="numpy not installed",@@ -2485,6 +2499,47 @@ class TestPandasTableManager(unittest.TestCase):assert json_data == [{"value": "(1.0,)"}, {"value": None}]+ def test_to_json_str_preserves_all_null_extension_nan_for_frontend(+ self,+ ) -> None:+ series = pd.Series([None], dtype="category")+ manager = self.factory.create()(series.to_frame(name="value"))++ assert manager.to_json_str(strict_json=False) == '[{"value":NaN}]'++ def test_to_json_str_preserves_mixed_extension_nan_for_frontend(+ self,+ ) -> None:+ series = pd.Series([7, None], dtype="category")+ manager = self.factory.create()(series.to_frame(name="value"))++ assert manager.to_json_str(strict_json=False) == (+ '[{"value":7},{"value":NaN}]'+ )++ def test_to_json_str_converts_extension_nan_for_export(self) -> None:+ series = pd.Series(["kept", None], dtype="category")+ manager = self.factory.create()(series.to_frame(name="value"))++ payload = manager.to_json_str(strict_json=True)++ assert payload == '[{"value":"kept"},{"value":null}]'+ assert msgspec.json.decode(payload) == [+ {"value": "kept"},+ {"value": None},+ ]++ def test_to_json_str_converts_all_null_extension_nan_for_export(+ self,+ ) -> None:+ series = pd.Series([None], dtype="category")+ manager = self.factory.create()(series.to_frame(name="value"))++ payload = manager.to_json_str(strict_json=True)++ assert payload == '[{"value":null}]'+ assert msgspec.json.decode(payload) == [{"value": None}]+def test_to_arrow_ipc_fallback_for_unsupported_extension_dtype(self,) -> None: