Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(table): preserve pandas sentinels by output mode (#10618)

Kiran Gadhavef470f31fc5eb
diff --git a/frontend/src/components/data-table/__tests__/columns.test.tsx b/frontend/src/components/data-table/__tests__/columns.test.tsx
index a2561ac1fb..b77d62f5d1 100644
--- a/frontend/src/components/data-table/__tests__/columns.test.tsx
+++ b/frontend/src/components/data-table/__tests__/columns.test.tsx
@@ -788,13 +788,13 @@ describe("LocaleNumber", () => {
});
describe("renderCellValue with string + edge whitespace", () => {
- const createMockStringColumn = () =>
+ const createMockStringColumn = (dtype = "object") =>
({
id: "desc",
columnDef: {
meta: {
dataType: "string" as const,
- dtype: "object",
+ dtype,
},
},
getColumnFormatting: () => undefined,
@@ -851,6 +851,21 @@ describe("renderCellValue with string + edge whitespace", () => {
}
});
+ it("renders pandas timedelta NaT as a sentinel", () => {
+ const mockColumn = createMockStringColumn("timedelta64[us]");
+ const result = renderCellValue({
+ column: mockColumn,
+ renderValue: () => "NaT",
+ getValue: () => "NaT",
+ selectCell: undefined,
+ cellStyles: "",
+ });
+
+ const { container } = renderWithProviders(result);
+
+ expect(container.querySelector('[aria-label="Not a Time"]')).not.toBeNull();
+ });
+
it("renders edge whitespace markers and still detects the URL in the middle", () => {
const mockColumn = createMockStringColumn();
const value = " https://example.com ";
diff --git a/frontend/src/components/data-table/__tests__/filter-by-values-picker.test.tsx b/frontend/src/components/data-table/__tests__/filter-by-values-picker.test.tsx
index 639c388db4..25d7c91f01 100644
--- a/frontend/src/components/data-table/__tests__/filter-by-values-picker.test.tsx
+++ b/frontend/src/components/data-table/__tests__/filter-by-values-picker.test.tsx
@@ -10,10 +10,10 @@ beforeAll(() => {
};
});
-function mockColumn(): Column<unknown, unknown> {
+function mockColumn(dtype?: string): Column<unknown, unknown> {
return {
id: "name",
- columnDef: { meta: { dataType: "string" } },
+ columnDef: { meta: { dataType: "string", dtype } },
} as unknown as Column<unknown, unknown>;
}
@@ -27,6 +27,19 @@ async function calculateTopK() {
}
describe("FilterByValuesList — creatable", () => {
+ it("renders pandas timedelta NaT as a sentinel", async () => {
+ render(
+ <FilterByValuesList
+ column={mockColumn("timedelta64[us]")}
+ calculateTopKRows={async () => ({ data: [["NaT", 1]] })}
+ chosenValues={new Set()}
+ onChange={vi.fn()}
+ />,
+ );
+
+ expect(await screen.findByLabelText("Not a Time")).toBeInTheDocument();
+ });
+
it("shows '+ Add \"X\"' item when creatable and query is non-empty", async () => {
const onChange = vi.fn();
render(
diff --git a/frontend/src/components/data-table/__tests__/utils.test.ts b/frontend/src/components/data-table/__tests__/utils.test.ts
index 0fafcdf232..7dc08b2c04 100644
--- a/frontend/src/components/data-table/__tests__/utils.test.ts
+++ b/frontend/src/components/data-table/__tests__/utils.test.ts
@@ -301,6 +301,8 @@ describe("detectSentinel", () => {
it("should not match NaT in non-temporal columns", () => {
expect(detectSentinel("NaT", "string")).toBeNull();
+ expect(detectSentinel("NaT", "string", "object")).toBeNull();
+ expect(detectSentinel("NaT", "string", "category")).toBeNull();
});
it("should match NaT in temporal columns", () => {
@@ -313,6 +315,26 @@ describe("detectSentinel", () => {
value: "NaT",
});
});
+
+ it("should match NaT in pandas timedelta and period columns", () => {
+ expect(detectSentinel("NaT", "string", "timedelta64[us]")).toEqual({
+ type: "nat",
+ value: "NaT",
+ });
+ expect(detectSentinel("NaT", "unknown", "period[M]")).toEqual({
+ type: "nat",
+ value: "NaT",
+ });
+ });
+
+ it("should match NaT in Narwhals duration columns", () => {
+ expect(detectSentinel("NaT", "number", "Duration(time_unit='us')")).toEqual(
+ {
+ type: "nat",
+ value: "NaT",
+ },
+ );
+ });
});
function createMockTableWithMeta<TData>(rawData?: TData[]): Table<TData> {
diff --git a/frontend/src/components/data-table/columns.tsx b/frontend/src/components/data-table/columns.tsx
index 7818a5d001..1bd6793a44 100644
--- a/frontend/src/components/data-table/columns.tsx
+++ b/frontend/src/components/data-table/columns.tsx
@@ -634,7 +634,7 @@ export function renderCellValue<TData, TValue>({
// Sentinel values (null, whitespace, NaN, Infinity, NaT) rendered specially.
// Empty strings are left as-is
- const sentinel = detectSentinel(value, dataType);
+ const sentinel = detectSentinel(value, dataType, dtype);
if (sentinel && sentinel.type !== "empty-string") {
return (
<div onClick={selectCell} className={cellStyles}>
diff --git a/frontend/src/components/data-table/filter-by-values-picker.tsx b/frontend/src/components/data-table/filter-by-values-picker.tsx
index 0b21a8bce0..1570a49a19 100644
--- a/frontend/src/components/data-table/filter-by-values-picker.tsx
+++ b/frontend/src/components/data-table/filter-by-values-picker.tsx
@@ -278,6 +278,7 @@ export const FilterByValuesList = <TData, TValue>({
const sentinel = detectSentinel(
value,
column.columnDef.meta?.dataType,
+ column.columnDef.meta?.dtype,
);
return (
<CommandItem
diff --git a/frontend/src/components/data-table/filters/types.ts b/frontend/src/components/data-table/filters/types.ts
index b8c958c8f6..0c6b862ed8 100644
--- a/frontend/src/components/data-table/filters/types.ts
+++ b/frontend/src/components/data-table/filters/types.ts
@@ -7,8 +7,10 @@ declare module "@tanstack/react-table" {
//allows us to define custom properties for our columns
interface ColumnMeta<TData extends RowData, TValue> {
rowHeader?: boolean;
- dtype?: string;
+ /** Normalized marimo type used for backend-independent UI behavior. */
dataType?: DataType;
+ /** Source-library type, such as pandas `timedelta64[ns]`. */
+ dtype?: string;
filterType?: FilterType;
minFractionDigits?: number;
width?: number;
diff --git a/frontend/src/components/data-table/row-viewer-panel/__tests__/row-viewer.test.tsx b/frontend/src/components/data-table/row-viewer-panel/__tests__/row-viewer.test.tsx
index 401b42e474..9a5e2aa79c 100644
--- a/frontend/src/components/data-table/row-viewer-panel/__tests__/row-viewer.test.tsx
+++ b/frontend/src/components/data-table/row-viewer-panel/__tests__/row-viewer.test.tsx
@@ -55,4 +55,20 @@ describe("RowExpandedPanel", () => {
expect(await screen.findByText("John")).toBeInTheDocument();
expect(await screen.findByText("30")).toBeInTheDocument();
});
+
+ it("renders pandas timedelta NaT as a sentinel", async () => {
+ renderWithProviders(
+ <RowViewerPanel
+ rowIdx={0}
+ setRowIdx={mockSetRowIdx}
+ totalRows={1}
+ fieldTypes={[["duration", ["string", "timedelta64[us]"]]]}
+ getRow={async () => ({ rows: [{ duration: "NaT" }] })}
+ isSelectable={false}
+ isRowSelected={false}
+ />,
+ );
+
+ expect(await screen.findByLabelText("Not a Time")).toBeInTheDocument();
+ });
});
diff --git a/frontend/src/components/data-table/row-viewer-panel/row-viewer.tsx b/frontend/src/components/data-table/row-viewer-panel/row-viewer.tsx
index e701027638..f9f3864d33 100644
--- a/frontend/src/components/data-table/row-viewer-panel/row-viewer.tsx
+++ b/frontend/src/components/data-table/row-viewer-panel/row-viewer.tsx
@@ -189,7 +189,7 @@ export const RowViewerPanel: React.FC<RowViewerPanelProps> = ({
</TableRow>
</TableHeader>
<TableBody>
- {fieldTypes?.map(([columnName, [dataType, _externalType]]) => {
+ {fieldTypes?.map(([columnName, [dataType, externalType]]) => {
const columnValue = rowValues[columnName];
if (!inSearchQuery({ columnName, columnValue, searchQuery })) {
@@ -201,6 +201,7 @@ export const RowViewerPanel: React.FC<RowViewerPanelProps> = ({
columnDef: {
meta: {
dataType,
+ dtype: externalType,
},
},
getColumnFormatting: () => undefined,
diff --git a/frontend/src/components/data-table/types.ts b/frontend/src/components/data-table/types.ts
index 13d14ec0c9..9741a57058 100644
--- a/frontend/src/components/data-table/types.ts
+++ b/frontend/src/components/data-table/types.ts
@@ -53,7 +53,12 @@ export type ColumnHeaderStats = Record<
export type FieldTypesWithExternalType = [
columnName: string,
- [dataType: DataType, externalType: string],
+ [
+ // Normalized marimo type, such as "string" or "datetime".
+ dataType: DataType,
+ // Source-library type, such as pandas "timedelta64[ns]".
+ externalType: string,
+ ],
][];
// Ordered map of column name -> data type.
//
diff --git a/frontend/src/components/data-table/utils.ts b/frontend/src/components/data-table/utils.ts
index 330902599d..d5e7e3f41a 100644
--- a/frontend/src/components/data-table/utils.ts
+++ b/frontend/src/components/data-table/utils.ts
@@ -154,14 +154,39 @@ const NUMERIC_STRING_SPECIALS: Record<string, StringValueSentinelType> = {
"-inf": "negative-infinity",
};
+// These source-library dtypes are temporal, but their normalized marimo data
+// types are "string", "number", or "unknown" instead of a temporal type.
+const NAT_DTYPE_PREFIXES = ["timedelta64[", "period[", "duration"] as const;
+
+function isNaTType(
+ dataType: DataType | undefined,
+ dtype: string | undefined,
+): boolean {
+ if (isTemporalType(dataType)) {
+ return true;
+ }
+ if (!dtype) {
+ return false;
+ }
+
+ const normalizedDtype = dtype.toLowerCase();
+ return NAT_DTYPE_PREFIXES.some((prefix) =>
+ normalizedDtype.startsWith(prefix),
+ );
+}
+
/**
* Detect if a cell value is a sentinel (null, empty string, whitespace,
* NaN, infinity, NaT). Column-type-dependent sentinels (string "NaN",
- * "NaT", etc.) are matched based on `dataType`.
+ * "NaT", etc.) are matched based on `dataType` and `dtype`.
+ *
+ * `dataType` is marimo's normalized, backend-independent type. `dtype` is
+ * the source library's more specific type string.
*/
export function detectSentinel(
value: unknown,
dataType: DataType | undefined,
+ dtype?: string,
): CellValueSentinel | null {
if (value == null) {
return { type: "null", value };
@@ -181,8 +206,7 @@ export function detectSentinel(
return { type, value };
}
}
- // String "NaT" in a temporal column = pandas Not-a-Time sentinel
- if (isTemporalType(dataType) && value === "NaT") {
+ if (value === "NaT" && isNaTType(dataType, dtype)) {
return { type: "nat", value };
}
return null;
diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.py
index 31246dc4a7..0b13786814 100644
--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py
+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py
@@ -278,12 +278,24 @@ class PandasTableManagerFactory(TableManagerFactory):
strict_json: bool = False,
ensure_ascii: bool = True,
) -> str:
+ """Serialize pandas rows for frontend display or strict export.
+
+ The frontend path can emit `NaN`, `Infinity`, `-Infinity`,
+ and typed `NaT` sentinels. The frontend parser and table
+ renderer display these values.
+
+ The strict path converts values that standard JSON does not
+ support. Pandas encodes these values as JSON `null`.
+ """
+
def to_json(
result: pd.DataFrame,
) -> list[dict[str, Any]] | str:
- """
- to_dict preserves nans, infs and is more accurate than to_json.
- By default, we use to_dict unless strict_json is True
+ """Serialize the prepared dataframe.
+
+ The frontend path intentionally preserves non-finite
+ sentinels through `to_dict`. The strict export path uses
+ pandas JSON, which converts them to JSON `null`.
"""
if strict_json:
try:
@@ -345,7 +357,13 @@ class PandasTableManagerFactory(TableManagerFactory):
if is_timedelta64_dtype(
dtype
) or is_timedelta64_ns_dtype(dtype):
- result[col] = _stringify_preserving_nulls(series)
+ # Preserve the NaT sentinel for frontend rendering;
+ # strict exports require JSON nulls.
+ result[col] = (
+ _stringify_preserving_nulls(series)
+ if strict_json
+ else series.apply(str)
+ )
if is_object_dtype(dtype):
# Convert date objects to the YYYY-MM-DD display form.
inferred_dtype = self._infer_dtype(col)
diff --git a/marimo/_plugins/ui/_impl/tables/table_manager.py b/marimo/_plugins/ui/_impl/tables/table_manager.py
index 3660537e5d..93a5e5fab8 100644
--- a/marimo/_plugins/ui/_impl/tables/table_manager.py
+++ b/marimo/_plugins/ui/_impl/tables/table_manager.py
@@ -125,12 +125,20 @@ class TableManager(abc.ABC, Generic[T]):
strict_json: bool = False,
ensure_ascii: bool = True,
) -> str:
- pass
+ """Serialize rows for frontend display or strict export.
+
+ `strict_json=False` requests output for the frontend special-value
+ parser. The output can contain bare non-finite numbers. A strict JSON
+ parser can reject this output.
+
+ `strict_json=True` requests standards-compliant output for downloads
+ and external JSON consumers.
+ """
def to_json(
self,
format_mapping: FormatMapping | None = None,
- strict_json: bool = False, # Whether the result should be strictly JSON compliant (eg. nan -> null)
+ strict_json: bool = False,
encoding: str | None = "utf-8",
ensure_ascii: bool = True,
) -> bytes:
diff --git a/marimo/_smoke_tests/tables/pandas_null_stringify.py b/marimo/_smoke_tests/tables/pandas_null_stringify.py
index 51a786809e..cbb30fad12 100644
--- a/marimo/_smoke_tests/tables/pandas_null_stringify.py
+++ b/marimo/_smoke_tests/tables/pandas_null_stringify.py
@@ -40,7 +40,7 @@ def _(date, pd):
@app.cell
-def _(PandasTableManagerFactory, dataframe, json):
+def _(PandasTableManagerFactory, dataframe, json, pd):
manager = PandasTableManagerFactory.create()(dataframe)
json_data = json.loads(manager.to_json_str())
assert json_data == [
@@ -53,12 +53,25 @@ def _(PandasTableManagerFactory, dataframe, json):
},
{
"complex": None,
- "timedelta": None,
+ "timedelta": "NaT",
"date": None,
"bytes": None,
"extension": None,
},
]
+ strict_json_data = json.loads(manager.to_json_str(strict_json=True))
+ assert strict_json_data[1]["timedelta"] is None
+
+ all_null_extension = pd.Series([None], dtype="category").to_frame(
+ name="extension"
+ )
+ all_null_manager = PandasTableManagerFactory.create()(all_null_extension)
+ assert all_null_manager.to_json_str(strict_json=False) == (
+ '[{"extension":NaN}]'
+ )
+ assert all_null_manager.to_json_str(strict_json=True) == (
+ '[{"extension":null}]'
+ )
json_data
return
diff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
index dad7e7e902..aaccc09e16 100644
--- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py
+++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
@@ -11,6 +11,7 @@ from types import SimpleNamespace
from typing import Any
from unittest.mock import Mock, patch
+import msgspec
import narwhals.stable.v2 as nw
import pytest
@@ -367,7 +368,6 @@ class TestPandasTableManager(unittest.TestCase):
df = pd.DataFrame(
{
"complex": [1 + 2j, complex(nan, nan)],
- "timedelta": [pd.Timedelta(days=1), pd.NaT],
"date": [datetime.date(2020, 1, 1), None],
"bytes": [b"ab", None],
}
@@ -378,18 +378,32 @@ class TestPandasTableManager(unittest.TestCase):
assert json_data == [
{
"complex": "(1+2j)",
- "timedelta": "1 days 00:00:00",
"date": "2020-01-01",
"bytes": "b'ab'",
},
{
"complex": None,
- "timedelta": None,
"date": None,
"bytes": None,
},
]
+ def test_to_json_str_preserves_timedelta_nat_for_frontend(self) -> None:
+ df = pd.DataFrame({"value": [pd.Timedelta(days=1), pd.NaT]})
+ manager = self.factory.create()(df)
+
+ assert manager.to_json_str(strict_json=False) == (
+ '[{"value":"1 days 00:00:00"},{"value":"NaT"}]'
+ )
+
+ def test_to_json_str_converts_timedelta_nat_for_export(self) -> None:
+ df = pd.DataFrame({"value": [pd.Timedelta(days=1), pd.NaT]})
+ manager = self.factory.create()(df)
+
+ assert manager.to_json_str(strict_json=True) == (
+ '[{"value":"1 days 00:00:00"},{"value":null}]'
+ )
+
@pytest.mark.skipif(
not DependencyManager.numpy.has(),
reason="numpy not installed",
@@ -2485,6 +2499,47 @@ class TestPandasTableManager(unittest.TestCase):
assert json_data == [{"value": "(1.0,)"}, {"value": None}]
+ def test_to_json_str_preserves_all_null_extension_nan_for_frontend(
+ self,
+ ) -> None:
+ series = pd.Series([None], dtype="category")
+ manager = self.factory.create()(series.to_frame(name="value"))
+
+ assert manager.to_json_str(strict_json=False) == '[{"value":NaN}]'
+
+ def test_to_json_str_preserves_mixed_extension_nan_for_frontend(
+ self,
+ ) -> None:
+ series = pd.Series([7, None], dtype="category")
+ manager = self.factory.create()(series.to_frame(name="value"))
+
+ assert manager.to_json_str(strict_json=False) == (
+ '[{"value":7},{"value":NaN}]'
+ )
+
+ def test_to_json_str_converts_extension_nan_for_export(self) -> None:
+ series = pd.Series(["kept", None], dtype="category")
+ manager = self.factory.create()(series.to_frame(name="value"))
+
+ payload = manager.to_json_str(strict_json=True)
+
+ assert payload == '[{"value":"kept"},{"value":null}]'
+ assert msgspec.json.decode(payload) == [
+ {"value": "kept"},
+ {"value": None},
+ ]
+
+ def test_to_json_str_converts_all_null_extension_nan_for_export(
+ self,
+ ) -> None:
+ series = pd.Series([None], dtype="category")
+ manager = self.factory.create()(series.to_frame(name="value"))
+
+ payload = manager.to_json_str(strict_json=True)
+
+ assert payload == '[{"value":null}]'
+ assert msgspec.json.decode(payload) == [{"value": None}]
+
def test_to_arrow_ipc_fallback_for_unsupported_extension_dtype(
self,
) -> None: