Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

feat(data): allow changing column data types (#3834)

Igor Dykhta0ba38a30e010
diff --git a/src/actions/src/action-types.ts b/src/actions/src/action-types.ts
index 8d9e3d416..8f2ccef12 100644
--- a/src/actions/src/action-types.ts
+++ b/src/actions/src/action-types.ts
@@ -82,6 +82,7 @@ export const ActionTypes = {
UPDATE_DATASET_PROPS: `${ACTION_PREFIX}UPDATE_DATASET_PROPS`,
UPDATE_DATASET: `${ACTION_PREFIX}UPDATE_DATASET`,
RENAME_TABLE_COLUMN: `${ACTION_PREFIX}RENAME_TABLE_COLUMN`,
+ CHANGE_TABLE_COLUMN_TYPE: `${ACTION_PREFIX}CHANGE_TABLE_COLUMN_TYPE`,
DELETE_TABLE_COLUMN: `${ACTION_PREFIX}DELETE_TABLE_COLUMN`,
TOGGLE_FILTER_ANIMATION: `${ACTION_PREFIX}TOGGLE_FILTER_ANIMATION`,
UPDATE_FILTER_ANIMATION_SPEED: `${ACTION_PREFIX}UPDATE_FILTER_ANIMATION_SPEED`,
diff --git a/src/actions/src/vis-state-actions.ts b/src/actions/src/vis-state-actions.ts
index ec68c1d64..c0870b0cc 100644
--- a/src/actions/src/vis-state-actions.ts
+++ b/src/actions/src/vis-state-actions.ts
@@ -1372,6 +1372,7 @@ export type LoadColumnStatsSuccessUpdaterAction = {
fieldName: string;
result: any;
filterProps?: any;
+ fieldType?: string;
};
/**
@@ -1383,7 +1384,8 @@ export function loadColumnStatsSuccess(
dataId: string,
fieldName: string,
result: any,
- filterProps?: any
+ filterProps?: any,
+ request?: {fieldType: string}
): Merge<
LoadColumnStatsSuccessUpdaterAction,
{type: typeof ActionTypes.LOAD_COLUMN_STATS_SUCCESS}
@@ -1393,7 +1395,8 @@ export function loadColumnStatsSuccess(
dataId,
fieldName,
result,
- filterProps
+ filterProps,
+ ...request
};
}
@@ -1401,6 +1404,7 @@ export type LoadColumnStatsErrorUpdaterAction = {
dataId: string;
fieldName: string;
error: Error;
+ fieldType?: string;
};
/**
@@ -1411,13 +1415,15 @@ export type LoadColumnStatsErrorUpdaterAction = {
export function loadColumnStatsError(
dataId: string,
fieldName: string,
- error: Error
+ error: Error,
+ request?: {fieldType: string}
): Merge<LoadColumnStatsErrorUpdaterAction, {type: typeof ActionTypes.LOAD_COLUMN_STATS_ERROR}> {
return {
type: ActionTypes.LOAD_COLUMN_STATS_ERROR,
dataId,
fieldName,
- error
+ error,
+ ...request
};
}
@@ -1582,6 +1588,33 @@ export function renameTableColumn(
};
}
+export type ChangeTableColumnTypeUpdaterAction = {
+ dataId: string;
+ fieldName: string;
+ newType: string;
+};
+/**
+ * Convert every value in a dataset column and update its field type atomically.
+ * Invalid values leave the dataset unchanged and produce an error notification.
+ * @param dataId - dataset id
+ * @param fieldName - column name
+ * @param newType - target kepler.gl field type
+ * @returns action
+ * @public
+ */
+export function changeTableColumnType(
+ dataId: string,
+ fieldName: string,
+ newType: string
+): Merge<ChangeTableColumnTypeUpdaterAction, {type: typeof ActionTypes.CHANGE_TABLE_COLUMN_TYPE}> {
+ return {
+ type: ActionTypes.CHANGE_TABLE_COLUMN_TYPE,
+ dataId,
+ fieldName,
+ newType
+ };
+}
+
export type DeleteTableColumnUpdaterAction = {
dataId: string;
fieldName: string;
diff --git a/src/components/src/common/data-table/header-cell.spec.tsx b/src/components/src/common/data-table/header-cell.spec.tsx
index da2704bfc..dd594ea27 100644
--- a/src/components/src/common/data-table/header-cell.spec.tsx
+++ b/src/components/src/common/data-table/header-cell.spec.tsx
@@ -37,27 +37,41 @@ const HeaderCell = HeaderCellFactory(
const columns = ['region', 'pop'];
const colMeta = {
- region: {name: 'region', type: 'string'},
- pop: {name: 'pop', type: 'integer'}
+ region: {name: 'region', type: 'string', colIdx: 0},
+ pop: {name: 'pop', type: 'integer', colIdx: 1}
};
function ColumnHeader({
renameTableColumn,
+ changeTableColumnType,
deleteTableColumn,
- createFilterFromColumn
+ createFilterFromColumn,
+ metadata = colMeta,
+ typeChangeValues
}: {
renameTableColumn?: (column: string, newName: string) => void;
+ changeTableColumnType?: (column: string, newType: string) => void;
deleteTableColumn?: (column: string) => void;
createFilterFromColumn?: (column: string) => void;
+ metadata?: typeof colMeta;
+ typeChangeValues?: unknown[];
}) {
const [moreOptionsColumn, setMoreOptionsColumn] = useState<string | null>(null);
const props = {
- colMeta,
+ colMeta: metadata,
+ typeChangeDataContainer: typeChangeValues
+ ? {
+ numRows: () => typeChangeValues.length,
+ numColumns: () => 1,
+ valueAt: (rowIndex: number) => typeChangeValues[rowIndex]
+ }
+ : undefined,
sortColumn: {},
pinTableColumn: jest.fn(),
copyTableColumn: jest.fn(),
sortTableColumn: jest.fn(),
renameTableColumn,
+ changeTableColumnType,
deleteTableColumn,
createFilterFromColumn
} as unknown as HeaderCellProps['props'];
@@ -178,4 +192,84 @@ describe('column header menu', () => {
fireEvent.click(screen.getByText('Create Filter'));
expect(createFilterFromColumn).toHaveBeenCalledWith('region');
});
+
+ test('changes the column data type from the column menu', () => {
+ const changeTableColumnType = jest.fn();
+ const {container} = renderWithTheme(
+ <ColumnHeader changeTableColumnType={changeTableColumnType} renameTableColumn={jest.fn()} />
+ );
+
+ openMenu(container);
+ fireEvent.click(screen.getByText('Change Data Type'));
+ expect(screen.queryByText('Boolean')).toBeNull();
+ expect(screen.queryByText('GeoJSON')).toBeNull();
+ fireEvent.click(screen.getByText('Integer'));
+ expect(changeTableColumnType).toHaveBeenCalledWith('region', 'integer');
+ });
+
+ test.each([false, true])(
+ 'does not scan the existing menu (type changes enabled: %s)',
+ enabled => {
+ const readValue = jest.fn(() => '10');
+ const values = [null];
+ Object.defineProperty(values, 0, {get: readValue});
+ const {container} = renderWithTheme(
+ <ColumnHeader
+ changeTableColumnType={enabled ? jest.fn() : undefined}
+ typeChangeValues={values}
+ />
+ );
+
+ openMenu(container);
+ expect(readValue).not.toHaveBeenCalled();
+ expect(screen.getByText('Copy Column')).toBeTruthy();
+ if (enabled) {
+ fireEvent.click(screen.getByText('Change Data Type'));
+ expect(readValue).toHaveBeenCalled();
+ } else {
+ expect(screen.queryByText('Change Data Type')).toBeNull();
+ }
+ }
+ );
+
+ test.each([
+ {type: 'string', values: ['ML', 'MW'], omitted: ['Integer', 'Real', 'Timestamp']},
+ {type: 'real', values: [3.1, 4], omitted: ['Integer', 'Timestamp']},
+ {type: 'real', values: [1_700_000_000.4, 1_700_000_001.6], omitted: ['Integer']},
+ {type: 'integer', values: [1, 2], omitted: ['Real', 'Timestamp']}
+ ])('offers compatible types for $type values $values', ({type, values, omitted}) => {
+ const changeTableColumnType = jest.fn();
+ const {container} = renderWithTheme(
+ <ColumnHeader
+ changeTableColumnType={changeTableColumnType}
+ metadata={{...colMeta, region: {...colMeta.region, type}}}
+ typeChangeValues={values}
+ />
+ );
+
+ openMenu(container);
+ fireEvent.click(screen.getByText('Change Data Type'));
+ omitted.forEach(label => expect(screen.queryByText(label)).toBeNull());
+ if (!omitted.includes('Timestamp')) expect(screen.getByText('Timestamp')).toBeTruthy();
+ fireEvent.click(screen.getByText('String'));
+ if (type === 'string') expect(changeTableColumnType).not.toHaveBeenCalled();
+ else expect(changeTableColumnType).toHaveBeenCalledWith('region', 'string');
+ });
+
+ test('hides type changes for field types outside the initial scope', () => {
+ const metadata = {
+ ...colMeta,
+ region: {...colMeta.region, type: 'boolean'}
+ };
+ const {container} = renderWithTheme(
+ <ColumnHeader
+ changeTableColumnType={jest.fn()}
+ metadata={metadata}
+ typeChangeValues={[true, false]}
+ />
+ );
+
+ openMenu(container);
+ expect(screen.queryByText('Change Data Type')).toBeNull();
+ });
});
diff --git a/src/components/src/common/data-table/header-cell.tsx b/src/components/src/common/data-table/header-cell.tsx
index 0b4e14c07..d9d251911 100644
--- a/src/components/src/common/data-table/header-cell.tsx
+++ b/src/components/src/common/data-table/header-cell.tsx
@@ -197,6 +197,8 @@ function HeaderCellFactory(
const {columnIndex, key, style} = cellInfo;
const {
colMeta,
+ dataContainer,
+ typeChangeDataContainer,
sortColumn,
sortTableColumn,
pinTableColumn,
@@ -207,6 +209,7 @@ function HeaderCellFactory(
dataId,
loadColumnStats,
renameTableColumn,
+ changeTableColumnType,
deleteTableColumn,
createFilterFromColumn
} = props;
@@ -383,6 +386,9 @@ function HeaderCellFactory(
copyTableColumn={onCopy}
setDisplayFormat={setColumnDisplayFormat ? onSetDisplayFormat : undefined}
renameTableColumn={renameTableColumn}
+ changeTableColumnType={changeTableColumnType}
+ typeChangeDataContainer={typeChangeDataContainer ?? dataContainer}
+ columnIndex={colMeta[column]?.colIdx ?? columns.indexOf(column)}
deleteTableColumn={deleteTableColumn}
createFilterFromColumn={createFilterFromColumn}
onStartRename={startRename}
diff --git a/src/components/src/common/data-table/index.tsx b/src/components/src/common/data-table/index.tsx
index d7873d767..71117fcf4 100644
--- a/src/components/src/common/data-table/index.tsx
+++ b/src/components/src/common/data-table/index.tsx
@@ -449,6 +449,8 @@ export interface DataTableProps {
fixedWidth?: number | null;
theme?: any;
dataContainer: DataContainerInterface | null;
+ /** Unfiltered source data used to validate whole-column type changes. */
+ typeChangeDataContainer?: DataContainerInterface | null;
fixedHeight?: number | null;
colMeta: ColMeta;
sortColumn: SortColumn;
@@ -458,6 +460,7 @@ export interface DataTableProps {
setColumnDisplayFormat?: (formats: {[key: string]: string}) => void;
copyTableColumn: (column: string) => void;
renameTableColumn?: (column: string, newName: string) => void;
+ changeTableColumnType?: (column: string, newType: string) => void;
deleteTableColumn?: (column: string) => void;
createFilterFromColumn?: (column: string) => void;
sortOrder?: number[] | null;
diff --git a/src/components/src/common/data-table/option-dropdown.tsx b/src/components/src/common/data-table/option-dropdown.tsx
index b7d43216d..710807df2 100644
--- a/src/components/src/common/data-table/option-dropdown.tsx
+++ b/src/components/src/common/data-table/option-dropdown.tsx
@@ -1,19 +1,21 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import React, {useCallback, useState} from 'react';
+import React, {useCallback, useMemo, useState} from 'react';
import styled from 'styled-components';
import Portaled from '../portaled';
import DropdownList from '../item-selector/dropdown-list';
import {
+ ALL_FIELD_TYPES,
SORT_ORDER,
TABLE_OPTION,
TABLE_OPTION_LIST,
TooltipFormat,
TableOption
} from '@kepler.gl/constants';
-import {getFieldFormatLabels} from '@kepler.gl/utils';
+import {DataContainerInterface, getFieldFormatLabels} from '@kepler.gl/utils';
import {ColMeta} from '@kepler.gl/types';
+import {getCompatibleColumnTypes} from '@kepler.gl/table';
import {
ArrowDown,
ArrowUp,
@@ -166,11 +168,23 @@ export interface OptionDropdownProps {
isSorted?: string;
isPinned?: boolean;
renameTableColumn?: (column: string, newName: string) => void;
+ changeTableColumnType?: (column: string, newType: string) => void;
+ typeChangeDataContainer?: DataContainerInterface | null;
+ columnIndex?: number;
deleteTableColumn?: (column: string) => void;
createFilterFromColumn?: (column: string) => void;
onStartRename?: () => void;
}
+const COLUMN_TYPE_OPTIONS = [
+ {type: ALL_FIELD_TYPES.integer, display: 'Integer'},
+ {type: ALL_FIELD_TYPES.real, display: 'Real'},
+ {type: ALL_FIELD_TYPES.string, display: 'String'},
+ {type: ALL_FIELD_TYPES.timestamp, display: 'Timestamp'}
+];
+
+type ColumnTypeOption = (typeof COLUMN_TYPE_OPTIONS)[number];
+
const OptionDropdown = (props: OptionDropdownProps) => {
const {
isOpened,
@@ -182,12 +196,30 @@ const OptionDropdown = (props: OptionDropdownProps) => {
hideTableColumn,
copyTableColumn,
setDisplayFormat,
+ changeTableColumnType,
+ typeChangeDataContainer,
+ columnIndex = -1,
deleteTableColumn,
createFilterFromColumn,
onStartRename
} = props;
const [showFormatter, setShowFormatter] = useState(false);
+ const [showTypeSelector, setShowTypeSelector] = useState(false);
const [confirmDelete, setConfirmDelete] = useState(false);
+ const currentType = colMeta[column]?.type;
+ const columnTypeOptions = useMemo<ColumnTypeOption[]>(() => {
+ if (!COLUMN_TYPE_OPTIONS.some(option => option.type === currentType)) {
+ return [];
+ }
+ if (!showTypeSelector || !changeTableColumnType || !typeChangeDataContainer) {
+ return COLUMN_TYPE_OPTIONS;
+ }
+ const compatibleTypes = new Set(
+ getCompatibleColumnTypes(typeChangeDataContainer, columnIndex, currentType)
+ );
+ return COLUMN_TYPE_OPTIONS.filter(option => compatibleTypes.has(option.type));
+ }, [changeTableColumnType, columnIndex, currentType, showTypeSelector, typeChangeDataContainer]);
+ const hasCompatibleAlternative = columnTypeOptions.some(option => option.type !== currentType);
const onOptionSelected: (v: TableOption) => void = useCallback(
({value}) => {
switch (value) {
@@ -218,6 +250,9 @@ const OptionDropdown = (props: OptionDropdownProps) => {
case TABLE_OPTION.RENAME:
onStartRename?.();
break;
+ case TABLE_OPTION.CHANGE_TYPE:
+ setShowTypeSelector(true);
+ return;
case TABLE_OPTION.CREATE_FILTER:
createFilterFromColumn?.(column);
break;
@@ -260,7 +295,8 @@ const OptionDropdown = (props: OptionDropdownProps) => {
// cant use conditions because it creates a circular dependency
// TODO: move condition clause out of default-settings TABLE_OPTION_LIST
const validToFormat = op.value !== TABLE_OPTION.FORMAT_COLUMN || formatLabels.length;
- return (!op.condition || op.condition(props)) && validToFormat;
+ const validToChangeType = op.value !== TABLE_OPTION.CHANGE_TYPE || hasCompatibleAlternative;
+ return (!op.condition || op.condition(props)) && validToFormat && validToChangeType;
}).map(op => ({
...op,
icon: TABLE_OPTION_LIST_ICONS[op.icon]
@@ -268,6 +304,7 @@ const OptionDropdown = (props: OptionDropdownProps) => {
const onClose = useCallback(() => {
setShowFormatter(false);
+ setShowTypeSelector(false);
setConfirmDelete(false);
toggleMoreOptions(column);
}, [column, toggleMoreOptions]);
@@ -312,6 +349,26 @@ const OptionDropdown = (props: OptionDropdownProps) => {
setDisplayFormat={setDisplayFormat}
onClose={onClose}
/>
+ <Portaled
+ left={120}
+ top={-10}
+ isOpened={Boolean(isOpened && showTypeSelector)}
+ onClose={onClose}
+ >
+ <StyledPopover className="column-type-popover">
+ <DropdownList
+ options={columnTypeOptions}
+ selectionIndex={columnTypeOptions.findIndex(option => option.type === currentType)}
+ displayOption={option => (option as ColumnTypeOption).display}
+ onOptionSelected={option => {
+ const {type} = option as ColumnTypeOption;
+ if (type !== currentType) changeTableColumnType?.(column, type);
+ onClose();
+ }}
+ light
+ />
+ </StyledPopover>
+ </Portaled>
</StyledOptionsDropdown>
</Portaled>
);
diff --git a/src/components/src/modal-container.tsx b/src/components/src/modal-container.tsx
index eabbe7537..71c8dda95 100644
--- a/src/components/src/modal-container.tsx
+++ b/src/components/src/modal-container.tsx
@@ -456,6 +456,7 @@ export default function ModalContainerFactory(
showAllTableColumns={visStateActions.showAllTableColumns}
copyTableColumn={visStateActions.copyTableColumn}
renameTableColumn={visStateActions.renameTableColumn}
+ changeTableColumnType={visStateActions.changeTableColumnType}
deleteTableColumn={visStateActions.deleteTableColumn}
createFilterFromColumn={(id, column) =>
visStateActions.createOrUpdateFilter(undefined, id, column)
diff --git a/src/components/src/modals/data-table-modal-column-edit.spec.tsx b/src/components/src/modals/data-table-modal-column-edit.spec.tsx
index 4e21bd00b..7bd60ebde 100644
--- a/src/components/src/modals/data-table-modal-column-edit.spec.tsx
+++ b/src/components/src/modals/data-table-modal-column-edit.spec.tsx
@@ -19,13 +19,22 @@ const tableProps: {current: Record<string, unknown> | null} = {current: null};
function StubDataTable(props: {
createFilterFromColumn?: (column: string) => void;
renameTableColumn?: (column: string, newName: string) => void;
+ changeTableColumnType?: (column: string, newType: string) => void;
deleteTableColumn?: (column: string) => void;
}) {
tableProps.current = props;
return (
- <button type="button" onClick={() => props.createFilterFromColumn?.('region')}>
- create-filter
- </button>
+ <>
+ <button type="button" onClick={() => props.createFilterFromColumn?.('region')}>
+ create-filter
+ </button>
+ <button
+ type="button"
+ onClick={() => props.changeTableColumnType?.('region', ALL_FIELD_TYPES.integer)}
+ >
+ change-type
+ </button>
+ </>
);
}
@@ -51,6 +60,7 @@ function makeDataset(type: string, hiddenColumns: string[] = []) {
function renderModal(type: string, hiddenColumns: string[] = []) {
const createFilterFromColumn = jest.fn();
const renameTableColumn = jest.fn();
+ const changeTableColumnType = jest.fn();
const deleteTableColumn = jest.fn();
const toggleSidePanel = jest.fn();
const toggleModal = jest.fn();
@@ -63,6 +73,7 @@ function renderModal(type: string, hiddenColumns: string[] = []) {
pinTableColumn={jest.fn()}
copyTableColumn={jest.fn()}
renameTableColumn={renameTableColumn}
+ changeTableColumnType={changeTableColumnType}
deleteTableColumn={deleteTableColumn}
createFilterFromColumn={createFilterFromColumn}
showDatasetTable={jest.fn()}
@@ -71,7 +82,13 @@ function renderModal(type: string, hiddenColumns: string[] = []) {
uiState={{} as never}
/>
);
- return {...view, createFilterFromColumn, toggleSidePanel, toggleModal};
+ return {
+ ...view,
+ createFilterFromColumn,
+ changeTableColumnType,
+ toggleSidePanel,
+ toggleModal
+ };
}
describe('data table column edits', () => {
@@ -83,22 +100,28 @@ describe('data table column edits', () => {
});
test('create filter opens the filter panel and closes the table', () => {
- const {createFilterFromColumn, toggleSidePanel, toggleModal} = renderModal('local');
+ const {createFilterFromColumn, changeTableColumnType, toggleSidePanel, toggleModal} =
+ renderModal('local');
expect(tableProps.current?.renameTableColumn).toEqual(expect.any(Function));
+ expect(tableProps.current?.changeTableColumnType).toEqual(expect.any(Function));
expect(tableProps.current?.deleteTableColumn).toEqual(expect.any(Function));
fireEvent.click(screen.getByRole('button', {name: 'create-filter'}));
expect(createFilterFromColumn).toHaveBeenCalledWith('cities', 'region');
expect(toggleSidePanel).toHaveBeenCalledWith('filter');
expect(toggleModal).toHaveBeenCalledWith(null);
+
+ fireEvent.click(screen.getByRole('button', {name: 'change-type'}));
+ expect(changeTableColumnType).toHaveBeenCalledWith('cities', 'region', 'integer');
});
- test('externally hosted datasets can create a filter but not rename or delete', () => {
+ test('externally hosted datasets cannot rename, re-type, or delete columns', () => {
renderModal('externally-hosted');
expect(tableProps.current?.createFilterFromColumn).toEqual(expect.any(Function));
expect(tableProps.current?.renameTableColumn).toBeUndefined();
+ expect(tableProps.current?.changeTableColumnType).toBeUndefined();
expect(tableProps.current?.deleteTableColumn).toBeUndefined();
});
diff --git a/src/components/src/modals/data-table-modal.tsx b/src/components/src/modals/data-table-modal.tsx
index cc8fd75db..4810eff34 100644
--- a/src/components/src/modals/data-table-modal.tsx
+++ b/src/components/src/modals/data-table-modal.tsx
@@ -254,6 +254,7 @@ interface DataTableModalProps {
showAllTableColumns?: (id: string) => void;
copyTableColumn: (id: string, column: string) => void;
renameTableColumn?: (id: string, column: string, newName: string) => void;
+ changeTableColumnType?: (id: string, column: string, newType: string) => void;
deleteTableColumn?: (id: string, column: string) => void;
createFilterFromColumn?: (id: string, column: string) => void;
datasets: Datasets;
@@ -285,6 +286,7 @@ function DataTableModalFactory(
showAllTableColumns,
copyTableColumn: copyTableColumnProp,
renameTableColumn,
+ changeTableColumnType,
deleteTableColumn,
createFilterFromColumn,
datasets,
@@ -435,6 +437,13 @@ function DataTableModalFactory(
[deleteTableColumn, dataId]
);
+ const handleChangeTableColumnType = useCallback(
+ (column: string, newType: string) => {
+ changeTableColumnType?.(dataId, column, newType);
+ },
+ [changeTableColumnType, dataId]
+ );
+
const handleCreateFilterFromColumn = useCallback(
(column: string) => {
createFilterFromColumn?.(dataId, column);
@@ -722,6 +731,7 @@ function DataTableModalFactory(
colMeta={colMeta}
cellSizeCache={cellSizeCache}
dataContainer={displayData.dataContainer}
+ typeChangeDataContainer={activeDataset.dataContainer}
pinnedColumns={(activeDataset.pinnedColumns || []).filter(
column => !hiddenColumns.includes(column)
)}
@@ -735,6 +745,9 @@ function DataTableModalFactory(
renameTableColumn={
canMutateColumns && renameTableColumn ? handleRenameTableColumn : undefined
}
+ changeTableColumnType={
+ canMutateColumns && changeTableColumnType ? handleChangeTableColumnType : undefined
+ }
deleteTableColumn={
canMutateColumns && deleteTableColumn ? handleDeleteTableColumn : undefined
}
diff --git a/src/constants/src/default-settings.ts b/src/constants/src/default-settings.ts
index 217604142..554c819d9 100644
--- a/src/constants/src/default-settings.ts
+++ b/src/constants/src/default-settings.ts
@@ -541,6 +541,7 @@ export const TABLE_OPTION = keyMirror({
FORMAT_COLUMN: null,
HIDE: null,
RENAME: null,
+ CHANGE_TYPE: null,
DELETE: null,
CREATE_FILTER: null
});
@@ -556,6 +557,7 @@ export type TableOption = {
| 'FORMAT_COLUMN'
| 'HIDE'
| 'RENAME'
+ | 'CHANGE_TYPE'
| 'DELETE'
| 'CREATE_FILTER';
display: string;
@@ -612,6 +614,12 @@ export const TABLE_OPTION_LIST: TableOption[] = [
icon: 'Edit',
condition: props => props.renameTableColumn
},
+ {
+ value: TABLE_OPTION.CHANGE_TYPE,
+ display: 'Change Data Type',
+ icon: 'Edit',
+ condition: props => props.changeTableColumnType
+ },
{
value: TABLE_OPTION.CREATE_FILTER,
display: 'Create Filter',
diff --git a/src/reducers/src/column-edit-reconcile.ts b/src/reducers/src/column-edit-reconcile.ts
index caf32d475..e6233f9dc 100644
--- a/src/reducers/src/column-edit-reconcile.ts
+++ b/src/reducers/src/column-edit-reconcile.ts
@@ -2,16 +2,29 @@
// Copyright contributors to the kepler.gl project
import {ChartConfig, LayerChartConfig} from '@kepler.gl/types';
+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
type NamedAxis = {
field?: {name: string; type?: string} | null;
title?: string | null;
+ aggregation?: string | null;
+ interval?: string | null;
};
+const BIN_AGGREGATIONS = new Set(['uniqueBin', 'numericBin', 'timeBin']);
+const TIME_FIELD_TYPES = new Set<string>([ALL_FIELD_TYPES.timestamp, ALL_FIELD_TYPES.date]);
+
+function binAggregationForType(type: string): string {
+ if (TIME_FIELD_TYPES.has(type)) {
+ return 'timeBin';
+ }
+ return type === ALL_FIELD_TYPES.real ? 'numericBin' : 'uniqueBin';
+}
+
function followFieldName(
name: string | null | undefined,
renames: Record<string, string>,
- alive: Set<string>
+ alive: ReadonlyMap<string, {type: string}>
): string | null {
if (!name) {
return null;
@@ -23,7 +36,7 @@ function followFieldName(
function followAxis<T extends NamedAxis>(
axis: T | undefined,
renames: Record<string, string>,
- alive: Set<string>
+ alive: ReadonlyMap<string, {type: string}>
): T | undefined {
if (!axis?.field?.name) {
return axis;
@@ -32,13 +45,22 @@ function followAxis<T extends NamedAxis>(
if (!name) {
return {...axis, field: null, title: null};
}
- if (name === axis.field.name) {
+ const field = alive.get(name);
+ const typeChanged = field?.type !== axis.field.type;
+ if (name === axis.field.name && !typeChanged) {
return axis;
}
+ const aggregation =
+ typeChanged && axis.aggregation && BIN_AGGREGATIONS.has(axis.aggregation)
+ ? binAggregationForType(field?.type || '')
+ : axis.aggregation;
+ const clearInterval = typeChanged && aggregation !== 'timeBin' && axis.interval !== undefined;
return {
...axis,
- field: {...axis.field, name},
- title: axis.title === axis.field.name ? name : axis.title
+ field: {...axis.field, name, type: field?.type},
+ title: axis.title === axis.field.name ? name : axis.title,
+ aggregation,
+ ...(clearInterval ? {interval: undefined} : {})
};
}
@@ -50,7 +72,7 @@ export function remapChartsForDataset(
charts: ChartConfig[] | undefined,
dataId: string,
renames: Record<string, string>,
- alive: Set<string>
+ alive: ReadonlyMap<string, {type: string}>
): ChartConfig[] {
return (charts || []).map(chart => {
if (chart.dataId !== dataId) {
diff --git a/src/reducers/src/column-edit-updaters.spec.ts b/src/reducers/src/column-edit-updaters.spec.ts
index 67c6e131c..01292894e 100644
--- a/src/reducers/src/column-edit-updaters.spec.ts
+++ b/src/reducers/src/column-edit-updaters.spec.ts
@@ -1,16 +1,54 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
+import {
+ ALL_FIELD_TYPES,
+ ANIMATION_WINDOW,
+ FILTER_TYPES,
+ PLOT_TYPES,
+ SORT_ORDER
+} from '@kepler.gl/constants';
import {VisStateActions} from '@kepler.gl/actions';
import {KeplerTable} from '@kepler.gl/table';
import SchemaManager, {VisState} from '@kepler.gl/schemas';
+import {drainTasksForTesting, errorTaskInTest, succeedTaskInTest} from '@kepler.gl/tasks';
+import type {PolygonFilter, RangeFilter, TimeRangeFilter} from '@kepler.gl/types';
import visStateReducer from './vis-state';
import {INITIAL_VIS_STATE} from './vis-state-updaters';
const reduceVisState = visStateReducer as (state: VisState, action: unknown) => VisState;
+function makeTable(id: string, data: Parameters<KeplerTable['updateSchema']>[0]) {
+ const table = new KeplerTable({
+ info: {id, label: id, type: 'local'},
+ color: [0, 0, 0]
+ });
+ return table.updateSchema(data);
+}
+
+function withTables(...tables: KeplerTable[]): VisState {
+ return {
+ ...INITIAL_VIS_STATE,
+ datasets: Object.fromEntries(tables.map(table => [table.id, table]))
+ };
+}
+
+function withFilter(state: VisState, dataId: string, column: string): VisState {
+ return reduceVisState(state, VisStateActions.createOrUpdateFilter(undefined, dataId, column));
+}
+
+function withSyncedFilter(primary: KeplerTable, secondary: KeplerTable, column: string): VisState {
+ return reduceVisState(
+ withFilter(withTables(primary, secondary), primary.id, column),
+ VisStateActions.setFilter(0, ['dataId', 'name'], [secondary.id, column], 1)
+ );
+}
+
+function changeType(state: VisState, dataId: string, column: string, type: string): VisState {
+ return reduceVisState(state, VisStateActions.changeTableColumnType(dataId, column, type));
+}
+
function makeCities() {
const table = new KeplerTable({
info: {id: 'cities', label: 'cities', type: 'local'},
@@ -93,6 +131,10 @@ function withCities(): VisState {
}
describe('column edits', () => {
+ beforeEach(() => {
+ drainTasksForTesting();
+ });
+
test('rename changes the field identity stored in layers, filters, and saved maps', () => {
let state = withCities();
state.datasets.cities.hiddenColumns = ['region', 'pop'];
@@ -171,6 +213,474 @@ describe('column edits', () => {
expect(kept.datasets.only.fields.map(field => field.name)).toEqual(['region']);
});
+ test('deleting an unrelated column preserves an inverted numeric histogram', () => {
+ const table = makeTable('data', {
+ fields: [
+ {name: 'unused', type: ALL_FIELD_TYPES.string},
+ {name: 'value', type: ALL_FIELD_TYPES.integer},
+ {name: 'category', type: ALL_FIELD_TYPES.string}
+ ],
+ rows: [
+ ['x', 1, 'A'],
+ ['x', 2, 'B'],
+ ['x', 3, 'A']
+ ]
+ });
+ let state = withFilter(withTables(table), 'data', 'category');
+ state = reduceVisState(state, VisStateActions.setFilter(0, 'value', ['A']));
+ state = withFilter(state, 'data', 'value');
+ state = reduceVisState(state, VisStateActions.setFilter(1, 'value', [2, 2]));
+ state = reduceVisState(state, VisStateActions.setFilter(1, 'inverted', true));
+ const histogramCount = () =>
+ (state.filters[1] as RangeFilter).bins?.data?.reduce((sum, bin) => sum + bin.count, 0);
+ expect(histogramCount()).toBe(2);
+
+ state = reduceVisState(state, VisStateActions.deleteTableColumn('data', 'unused'));
+
+ expect(histogramCount()).toBe(2);
+ expect(state.datasets.data.filteredIndex).toEqual([0, 2]);
+ expect(state.datasets.data.fields.map(field => field.type)).toEqual([
+ ALL_FIELD_TYPES.integer,
+ ALL_FIELD_TYPES.string
+ ]);
+ });
+
+ test('type change converts values and reconciles incompatible references', () => {
+ let state = withCities();
+ state = reduceVisState(
+ state,
+ VisStateActions.addLayer({
+ id: 'cities-heatmap',
+ type: 'heatmap',
+ config: {
+ dataId: 'cities',
+ label: 'cities heatmap',
+ color: [255, 0, 0],
+ columns: {lat: 'lat', lng: 'lng'},
+ weightField: {name: 'pop', type: 'integer'},
+ textLabel: [{field: {name: 'pop', type: 'integer'}}]
+ }
+ })
+ );
+ state = changeType(state, 'cities', 'pop', ALL_FIELD_TYPES.string);
+
+ const pop = state.datasets.cities.fields.find(field => field.name === 'pop');
+ expect(pop).toMatchObject({type: ALL_FIELD_TYPES.string, analyzerType: 'STRING', format: ''});
+ expect(state.datasets.cities.dataContainer.rowAsArray(0)[3]).toBe('10');
+ expect(state.layers).toHaveLength(2);
+ expect(
+ (state.layers.find(layer => layer.id === 'cities-heatmap')?.config as {weightField?: unknown})
+ .weightField
+ ).toBeNull();
+ expect(
+ state.layers.find(layer => layer.id === 'cities-heatmap')?.config.textLabel[0].field
+ ).toMatchObject({name: 'pop', type: 'string'});
+ expect(state.charts[0].yAxis?.field).toEqual({name: 'pop', type: 'string'});
+ });
+
+ test('type change preserves polygon filters targeting the dataset', () => {
+ let state = withCities();
+ const polygonFilter: PolygonFilter = {
+ id: 'polygon-filter',
+ dataId: ['cities'],
+ enabled: true,
+ fixedDomain: true,
+ view: 'side',
+ isAnimating: false,
+ speed: 1,
+ name: [],
+ type: FILTER_TYPES.polygon,
+ fieldIdx: [],
+ domain: null,
+ value: {
+ type: 'Feature',
+ id: 'polygon-selection',
+ properties: {filterId: 'polygon-filter'},
+ geometry: {
+ type: 'Polygon',
+ coordinates: [
+ [
+ [-123, 37],
+ [-122, 37],
+ [-122, 38],
+ [-123, 37]
+ ]
+ ]
+ }
+ },
+ plotType: {type: PLOT_TYPES.histogram},
+ yAxis: null,
+ layerId: ['cities-layer'],
+ gpu: false
+ };
+
+ state = {...state, filters: [...state.filters, polygonFilter]};
+ state = changeType(state, 'cities', 'pop', ALL_FIELD_TYPES.string);
+
+ expect(state.filters.find(filter => filter.id === polygonFilter.id)).toBe(polygonFilter);
+
+ state = reduceVisState(state, VisStateActions.deleteTableColumn('cities', 'lat'));
+
+ expect(state.layers).toHaveLength(0);
+ expect(state.filters.find(filter => filter.id === polygonFilter.id)).toBeUndefined();
+ expect(state.datasets.cities.filterRecord?.gpu).not.toContainEqual(
+ expect.objectContaining({id: polygonFilter.id})
+ );
+ expect(state.datasets.cities.filterRecord?.cpu).not.toContainEqual(
+ expect.objectContaining({id: polygonFilter.id})
+ );
+ });
+
+ test('type change preserves an existing sort on another column', () => {
+ let state = withCities();
+ state = reduceVisState(
+ state,
+ VisStateActions.sortTableColumn('cities', 'region', SORT_ORDER.ASCENDING)
+ );
+ expect(state.datasets.cities.sortColumn).toEqual({region: SORT_ORDER.ASCENDING});
+ expect(state.datasets.cities.sortOrder).toEqual([1, 0]);
+
+ state = changeType(state, 'cities', 'pop', ALL_FIELD_TYPES.string);
+
+ expect(state.datasets.cities.sortColumn).toEqual({region: SORT_ORDER.ASCENDING});
+ expect(state.datasets.cities.sortOrder).toEqual([1, 0]);
+ });
+
+ test('type change clears timeline state when a time filter becomes non-time', () => {
+ const events = makeTable('events', {
+ fields: [
+ {name: 'start', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'DATETIME'},
+ {name: 'amount', type: ALL_FIELD_TYPES.integer}
+ ],
+ rows: [
+ ['2024-01-01T00:00:00Z', 10],
+ ['2024-01-02T00:00:00Z', 20]
+ ]
+ });
+ let state = withFilter(withTables(events), 'events', 'start');
+ state = {
+ ...state,
+ filters: [
+ {
+ ...(state.filters[0] as TimeRangeFilter),
+ syncedWithLayerTimeline: true,
+ syncTimelineMode: 1 as const,
+ isAnimating: true,
+ animationWindow: ANIMATION_WINDOW.interval,
+ timeBins: {events: {day: []}},
+ lineChart: {
+ series: [],
+ yDomain: [0, 1],
+ xDomain: [0, 1],
+ aggregation: 'sum',
+ interval: 'day',
+ yAxis: 'amount'
+ },
+ timezone: 'UTC',
+ invertTrendColor: true,
+ showTimeDisplay: true,
+ yAxis: state.datasets.events.fields[1],
+ plotType: {type: PLOT_TYPES.lineChart, interval: 'day', aggregation: 'sum'}
+ }
+ ]
+ };
+
+ state = changeType(state, 'events', 'start', ALL_FIELD_TYPES.string);
+
+ const filter = state.filters[0];
+ expect(filter.type).toBe('multiSelect');
+ expect(filter).not.toHaveProperty('syncedWithLayerTimeline');
+ expect(filter).not.toHaveProperty('syncTimelineMode');
+ expect(filter).not.toHaveProperty('timeBins');
+ expect(filter).not.toHaveProperty('lineChart');
+ expect(filter).not.toHaveProperty('timezone');
+ expect(filter).not.toHaveProperty('invertTrendColor');
+ expect(filter).not.toHaveProperty('showTimeDisplay');
+ expect(filter).toMatchObject({
+ fixedDomain: false,
+ isAnimating: false,
+ animationWindow: ANIMATION_WINDOW.free,
+ yAxis: null,
+ plotType: {type: PLOT_TYPES.histogram}
+ });
+ });
+
+ test('type change detaches an incompatible synced filter member and refilters its dataset', () => {
+ const primary = makeTable('primary', {
+ fields: [{name: 'value', type: ALL_FIELD_TYPES.integer}],
+ rows: [[1], [2]]
+ });
+ const secondary = makeTable('secondary', {
+ fields: [{name: 'value', type: ALL_FIELD_TYPES.integer}],
+ rows: [[1], [3]]
+ });
+
+ let state = withSyncedFilter(primary, secondary, 'value');
+ expect(state.filters[0].dataId).toEqual(['primary', 'secondary']);
+ expect(state.datasets.secondary.filterRecord?.gpu).toHaveLength(1);
+
+ state = changeType(state, 'secondary', 'value', ALL_FIELD_TYPES.string);
+
+ expect(state.filters[0].dataId).toEqual(['primary']);
+ expect(state.filters[0].name).toEqual(['value']);
+ expect(state.filters[0].type).toBe('range');
+ expect(state.datasets.secondary.filterRecord).toMatchObject({
+ dynamicDomain: [],
+ fixedDomain: [],
+ cpu: [],
+ gpu: []
+ });
+ });
+
+ test('changing a synced secondary dataset does not reconcile the primary plot axis', () => {
+ const makeEvents = (id: string) =>
+ makeTable(id, {
+ fields: [
+ {name: 'start', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'TIME', format: 'x'},
+ {name: 'amount', type: ALL_FIELD_TYPES.integer}
+ ],
+ rows: [
+ [1_700_000_000, 10],
+ [1_700_000_020, 20]
+ ]
+ });
+ const primary = makeEvents('primary');
+ const secondary = makeEvents('secondary');
+ let state = withSyncedFilter(primary, secondary, 'start');
+ state = reduceVisState(
+ state,
+ VisStateActions.setFilterPlot(0, {yAxis: state.datasets.primary.fields[1]})
+ );
+
+ state = changeType(state, 'secondary', 'amount', ALL_FIELD_TYPES.string);
+
+ expect(state.filters[0].dataId).toEqual(['primary', 'secondary']);
+ expect(state.filters[0].yAxis).toMatchObject({name: 'amount', type: 'integer'});
+ expect(state.filters[0].plotType.type).toBe('lineChart');
+ });
+
+ test('detaching a synced primary dataset rebuilds the plot for the new primary', () => {
+ const primary = makeTable('primary', {
+ fields: [
+ {name: 'start', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'TIME', format: 'x'},
+ {name: 'amount', type: ALL_FIELD_TYPES.integer}
+ ],
+ rows: [
+ [1_700_000_000, 10],
+ [1_700_000_020, 20]
+ ]
+ });
+ const secondary = makeTable('secondary', {
+ fields: [{name: 'start', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'TIME', format: 'x'}],
+ rows: [[1_700_000_040], [1_700_000_060]]
+ });
+ let state = withSyncedFilter(primary, secondary, 'start');
+ state = reduceVisState(
+ state,
+ VisStateActions.setFilterPlot(0, {yAxis: state.datasets.primary.fields[1]})
+ );
+ expect(state.filters[0].lineChart).toBeDefined();
+
+ state = changeType(state, 'primary', 'start', ALL_FIELD_TYPES.string);
+
+ expect(state.filters[0].dataId).toEqual(['secondary']);
+ expect(state.filters[0].name).toEqual(['start']);
+ expect(state.filters[0].type).toBe('timeRange');
+ expect(state.filters[0].yAxis).toBeNull();
+ expect(state.filters[0].plotType.type).toBe('histogram');
+ expect(state.filters[0].lineChart).toBeUndefined();
+ });
+
+ test('grouped timeline series survive a rename and rebuild when group values collapse', () => {
+ const events = makeTable('events', {
+ fields: [
+ {name: 'start', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'TIME', format: 'x'},
+ {name: 'group', type: ALL_FIELD_TYPES.string},
+ {name: 'amount', type: ALL_FIELD_TYPES.integer}
+ ],
+ rows: [
+ [1_700_000_000, '01', 10],
+ [1_700_000_020, '1', 20]
+ ]
+ });
+ let state = withFilter(withTables(events), 'events', 'start');
+ state = reduceVisState(
+ state,
+ VisStateActions.setFilterPlot(0, {yAxis: state.datasets.events.fields[2]})
+ );
+ state = reduceVisState(
+ state,
+ VisStateActions.setFilterPlot(0, {
+ plotType: {groupBy: {fieldName: 'group', numGroups: 'ALL', groupOthers: false}}
+ })
+ );
+ const seriesBefore = state.filters[0].lineChart?.series as {names?: string[]};
+ expect(new Set(seriesBefore.names)).toEqual(new Set(['01', '1']));
+
+ state = reduceVisState(state, VisStateActions.renameTableColumn('events', 'group', 'category'));
+ expect(state.filters[0].lineChart?.series).toEqual(seriesBefore);
+ expect(state.filters[0].plotType.groupBy.fieldName).toBe('category');
+ expect(state.datasets.events.fields[1].type).toBe(ALL_FIELD_TYPES.string);
+
+ state = changeType(state, 'events', 'category', ALL_FIELD_TYPES.integer);
+
+ const namesAfter = (state.filters[0].lineChart?.series as {names?: string[]}).names;
+ expect(new Set(namesAfter)).toEqual(new Set(['1']));
+ });
+
+ test('type change clears an interval end field that is no longer a timestamp', () => {
+ const events = makeTable('events', {
+ fields: [
+ {name: 'start', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'TIME', format: 'x'},
+ {name: 'end', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'TIME', format: 'x'},
+ {name: 'value', type: ALL_FIELD_TYPES.integer}
+ ],
+ rows: [
+ [1_700_000_000, 1_700_086_400, 10],
+ [1_700_000_020, 1_700_172_800, 20]
+ ]
+ });
+ let state = withTables(events);
+ state = reduceVisState(
+ state,
+ VisStateActions.addChart({
+ id: 'events-chart',
+ title: 'Events',
+ type: 'barChart',
+ dataId: 'events',
+ applyFilters: true,
+ display: {},
+ chartDisplay: {},
+ xAxis: {
+ field: {name: 'end', type: 'timestamp'},
+ aggregation: 'timeBin',
+ interval: 'day',
+ title: 'end'
+ },
+ yAxis: {field: {name: 'value', type: 'integer'}, aggregation: 'sum', title: 'value'}
+ })
+ );
+ state = withFilter(state, 'events', 'start');
+ state = reduceVisState(state, VisStateActions.setFilter(0, 'endName', 'end', 0));
+ state = reduceVisState(
+ state,
+ VisStateActions.setFilterPlot(0, {yAxis: state.datasets.events.fields[2]})
+ );
+ expect(state.filters[0].endName).toEqual(['end']);
+ expect(state.filters[0].yAxis?.name).toBe('value');
+
+ state = changeType(state, 'events', 'value', ALL_FIELD_TYPES.string);
+ expect(state.filters[0].yAxis).toBeNull();
+ expect(state.filters[0].plotType.type).toBe('histogram');
+ expect(state.filters[0].endName).toEqual(['end']);
+ expect(state.charts[0].yAxis).toMatchObject({
+ field: {name: 'value', type: 'string'},
+ aggregation: 'sum'
+ });
+ let timeFilter = state.filters[0] as TimeRangeFilter;
+ const interval = timeFilter.plotType.interval;
+ const intervalBins = timeFilter.timeBins?.events?.[interval];
+ expect(intervalBins?.reduce((sum, bin) => sum + bin.count, 0)).toBeGreaterThan(2);
+
+ state = changeType(state, 'events', 'end', ALL_FIELD_TYPES.string);
+ expect(state.filters[0].type).toBe('timeRange');
+ expect(state.filters[0].endName).toBeUndefined();
+ expect(state.filters[0].endFieldIdx).toBeUndefined();
+ timeFilter = state.filters[0] as TimeRangeFilter;
+ const bins = timeFilter.timeBins?.events?.[interval];
+ expect(bins?.reduce((sum, bin) => sum + bin.count, 0)).toBe(2);
+ expect(state.charts[0].xAxis).toMatchObject({
+ field: {name: 'end', type: 'string'},
+ aggregation: 'uniqueBin'
+ });
+ expect(state.charts[0].xAxis?.interval).toBeUndefined();
+ });
+
+ test('invalid type change is atomic and schedules a precise error', () => {
+ const state = withCities();
+ const next = changeType(state, 'cities', 'region', ALL_FIELD_TYPES.integer);
+
+ expect(next.datasets.cities).toBe(state.datasets.cities);
+ expect(next.datasets.cities.fields.find(field => field.name === 'region')?.type).toBe(
+ ALL_FIELD_TYPES.string
+ );
+ expect(next.datasets.cities.dataContainer.rowAsArray(0)[2]).toBe('west');
+
+ const tasks = drainTasksForTesting();
+ expect(tasks).toHaveLength(1);
+ const notification = succeedTaskInTest(tasks[0], undefined);
+ expect(notification.payload.message).toContain(
+ 'Cannot change “region” to integer: row 1 value “west” is not a safe integer'
+ );
+ expect(notification.payload.message).toContain('No values were changed.');
+ });
+
+ test('late column statistics cannot overwrite a changed column', () => {
+ let state = withCities();
+ state = reduceVisState(state, VisStateActions.loadColumnStats('cities', 'pop'));
+ const requestRevision = state.datasets.cities.dataRevision;
+ const [task] = drainTasksForTesting();
+ const result = {type: 'numeric', mean: 25};
+ const staleSuccess = succeedTaskInTest(task, {
+ result,
+ filterProps: {type: FILTER_TYPES.range}
+ });
+ expect(staleSuccess).toMatchObject({
+ fieldType: ALL_FIELD_TYPES.integer
+ });
+
+ state = changeType(state, 'cities', 'pop', ALL_FIELD_TYPES.string);
+ expect(state.datasets.cities.dataRevision).toBe(requestRevision + 1);
+ const changedState = state;
+
+ state = reduceVisState(state, staleSuccess);
+
+ expect(state).toBe(changedState);
+ const changedField = state.datasets.cities.fields.find(field => field.name === 'pop');
+ expect(changedField).toMatchObject({type: ALL_FIELD_TYPES.string});
+ expect(changedField).not.toHaveProperty('isLoadingStats');
+ expect(
+ state.datasets.cities.fields.find(field => field.name === 'pop')?.filterProps?.columnStats
+ ).not.toBe(result);
+ });
+
+ test.each(['refresh', 'append'])('pending statistics complete after a normal %s', update => {
+ let state = withCities();
+ const types = state.datasets.cities.fields.map(field => field.type);
+ state = reduceVisState(state, VisStateActions.loadColumnStats('cities', 'pop'));
+ const [task] = drainTasksForTesting();
+ state = reduceVisState(state, VisStateActions.pinTableColumn('cities', 'region'));
+ const row = [41, -75, 'north', 50];
+ state = reduceVisState(
+ state,
+ update === 'refresh'
+ ? VisStateActions.refreshDatasetSuccess('cities', {
+ notModified: false,
+ data: {fields: state.datasets.cities.fields, rows: [row]}
+ })
+ : VisStateActions.addToDataset('cities', [row])
+ );
+
+ const result = {type: 'numeric', mean: 25};
+ const successState = reduceVisState(
+ state,
+ succeedTaskInTest(task, {result, filterProps: {type: FILTER_TYPES.range}})
+ );
+ const logError = jest.spyOn(console, 'error').mockImplementation(() => undefined);
+ const errorState = reduceVisState(state, errorTaskInTest(task, new Error('stats failed')));
+ logError.mockRestore();
+
+ expect(state.datasets.cities.fields.map(field => field.type)).toEqual(types);
+ expect(successState.datasets.cities.fields[3]).toMatchObject({
+ isLoadingStats: false,
+ filterProps: {columnStats: result}
+ });
+ expect(errorState.datasets.cities.fields[3].isLoadingStats).toBe(false);
+ expect(state.datasets.cities.dataContainer.rowAsArray(update === 'refresh' ? 0 : 2)).toEqual(
+ row
+ );
+ });
+
test('duplicate rename and non-tabular datasets are ignored', () => {
const state = withCities();
const duplicate = reduceVisState(
@@ -197,6 +707,7 @@ describe('column edits', () => {
);
expect(unchanged).toBe(tiled);
expect(reduceVisState(tiled, VisStateActions.deleteTableColumn('tiles', 'pop'))).toBe(tiled);
+ expect(changeType(tiled, 'tiles', 'pop', ALL_FIELD_TYPES.string)).toBe(tiled);
});
test('externally hosted datasets ignore rename and delete', () => {
@@ -217,6 +728,7 @@ describe('column edits', () => {
reduceVisState(state, VisStateActions.renameTableColumn('remote', 'region', 'area'))
).toBe(state);
expect(reduceVisState(state, VisStateActions.deleteTableColumn('remote', 'pop'))).toBe(state);
+ expect(changeType(state, 'remote', 'pop', ALL_FIELD_TYPES.string)).toBe(state);
expect(state.datasets.remote.fields[0].displayName).toBe('Region');
});
});
diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts
index 1914221f8..a140b816e 100644
--- a/src/reducers/src/vis-state-updaters.ts
+++ b/src/reducers/src/vis-state-updaters.ts
@@ -103,6 +103,7 @@ import {
import {generateHashId, toArray, arrayMove} from '@kepler.gl/common-utils';
// Mergers
import {
+ ALL_FIELD_TYPES,
ANIMATION_WINDOW,
BASE_SPEED,
COMPARE_TYPES,
@@ -201,6 +202,8 @@ import {
Datasets,
KeplerTable,
FilterProps,
+ buildChangeColumnTypeData,
+ formatColumnTypeConversionError,
assignGpuChannel,
buildDeleteColumnData,
buildRenameColumnData,
@@ -4410,16 +4413,14 @@ function reconcileLayerWithNewSchema(
}
});
- // follow renamed text-label fields
+ // follow renamed or re-typed text-label fields
if (layer.config.textLabel?.length) {
- const textLabel = layer.config.textLabel.map(tl =>
- tl.field && renames[tl.field.name]
- ? {...tl, field: {...tl.field, name: renames[tl.field.name]}}
- : tl
- );
- if (textLabel.some((tl, i) => tl !== layer.config.textLabel[i])) {
- configPatch.textLabel = textLabel;
- }
+ configPatch.textLabel = layer.config.textLabel.map(tl => {
+ const name = tl.field?.name;
+ const field =
+ name && newTable.fields.find(candidate => candidate.name === (renames[name] ?? name));
+ return field && (field.name !== name || field.type !== tl.field?.type) ? {...tl, field} : tl;
+ });
}
// shallow-copy the layer so we never mutate redux state; the copy keeps the
@@ -4440,6 +4441,57 @@ function reconcileLayerWithNewSchema(
return validated && validated.hasAllColumns() ? validated : null;
}
+const FILTER_PLOT_Y_AXIS_TYPES = new Set<string>([ALL_FIELD_TYPES.integer, ALL_FIELD_TYPES.real]);
+
+function clearTimelineFilterState(filter: Filter): Filter {
+ const {
+ syncedWithLayerTimeline: _syncedWithLayerTimeline,
+ syncTimelineMode: _syncTimelineMode,
+ timeBins: _timeBins,
+ lineChart: _lineChart,
+ timezone: _timezone,
+ invertTrendColor: _invertTrendColor,
+ showTimeDisplay: _showTimeDisplay,
+ ...rest
+ } = filter as TimeRangeFilter;
+
+ return {
+ ...rest,
+ fixedDomain: false,
+ isAnimating: false,
+ animationWindow: ANIMATION_WINDOW.free,
+ yAxis: null,
+ plotType: {type: PLOT_TYPES.histogram}
+ } as Filter;
+}
+
+function reconcileTimelinePlotAfterPrimaryRemoval(
+ filter: Filter,
+ datasets: VisState['datasets']
+): Filter {
+ if (filter.type !== FILTER_TYPES.timeRange) {
+ return filter;
+ }
+
+ const primaryDataset = datasets[filter.dataId[0]];
+ const yAxis = filter.yAxis?.name
+ ? primaryDataset?.fields.find(
+ field => field.name === filter.yAxis?.name && FILTER_PLOT_Y_AXIS_TYPES.has(field.type)
+ ) ?? null
+ : null;
+ const groupByFieldName = filter.plotType?.groupBy?.fieldName;
+ const hasGroupByField = groupByFieldName
+ ? primaryDataset?.fields.some(field => field.name === groupByFieldName)
+ : true;
+ const plotType = {
+ ...filter.plotType,
+ ...(filter.yAxis && !yAxis ? {type: PLOT_TYPES.histogram} : {}),
+ ...(!hasGroupByField ? {groupBy: null} : {})
+ };
+
+ return {...filter, yAxis, plotType, lineChart: undefined};
+}
+
/**
* Update an existing dataset's schema (columns/fields) in place and reconcile
* everything that referenced the old columns: filters, layers, tooltip fields,
@@ -4470,7 +4522,9 @@ export function updateDatasetUpdater(
// 1. rebuild the table schema in place (same id/label/color/metadata)
const newTable = copyTableAndUpdate(existing, {});
+ const [sortedColumn, sortMode] = Object.entries(existing.sortColumn || {})[0] || [];
newTable.updateSchema(data);
+ newTable.dataRevision = existing.dataRevision + 1;
// The data container is rebuilt unsorted and unfiltered; reset the table's
// view state that indexes into the old container.
@@ -4478,15 +4532,20 @@ export function updateDatasetUpdater(
newTable.filterRecordCPU = undefined;
newTable.sortColumn = undefined;
newTable.sortOrder = null;
- const newFieldsByName = new Set(newTable.fields.map(f => f.name));
+ const newFieldsByName = new Map(newTable.fields.map(field => [field.name, field]));
newTable.pinnedColumns = (existing.pinnedColumns || [])
.map(name => renames[name] ?? name)
.filter(name => newFieldsByName.has(name));
newTable.hiddenColumns = (existing.hiddenColumns || [])
.map(name => renames[name] ?? name)
.filter(name => newFieldsByName.has(name));
+ const nextSortedColumn = sortedColumn ? renames[sortedColumn] ?? sortedColumn : null;
+ if (nextSortedColumn && sortMode && newFieldsByName.has(nextSortedColumn)) {
+ sortDatasetByColumn(newTable, nextSortedColumn, sortMode);
+ }
const newDatasets = {...state.datasets, [dataId]: newTable};
+ const datasetIdsToRefilter = new Set<string>([dataId]);
// 2. reconcile filters
let newFilters = state.filters
@@ -4496,57 +4555,92 @@ export function updateDatasetUpdater(
return filter;
}
+ // Polygon filters target layers rather than dataset fields, so a column
+ // type change should not reconcile them against the field list.
+ if (filter.type === FILTER_TYPES.polygon) {
+ return filter;
+ }
+
const oldName = filter.name[vi];
const newName = renames[oldName] ?? oldName;
const fieldIdx = newTable.getColumnFieldIdx(newName);
+ const nextFilterType = fieldIdx === -1 ? null : newTable.getColumnFilterProps(newName)?.type;
- if (fieldIdx === -1) {
- // column is gone: drop a single-dataset filter, splice it out of a synced one
- return filter.dataId.length === 1
- ? null
- : _removeFilterDataIdAtValueIndex(filter, vi, newDatasets);
+ if (
+ fieldIdx === -1 ||
+ (filter.dataId.length > 1 && nextFilterType && nextFilterType !== filter.type)
+ ) {
+ // Drop a missing field or detach an incompatible synced member.
+ if (filter.dataId.length === 1) {
+ return null;
+ }
+ filter.dataId.forEach(id => datasetIdsToRefilter.add(id));
+ const detachedFilter = _removeFilterDataIdAtValueIndex(filter, vi, newDatasets);
+ return vi === 0
+ ? reconcileTimelinePlotAfterPrimaryRemoval(detachedFilter, newDatasets)
+ : detachedFilter;
}
// column present (possibly renamed): re-resolve name/fieldIdx/domain
- const {filter: updated} = applyFilterFieldName(filter, newDatasets, dataId, newName, vi, {
+ let {filter: updated} = applyFilterFieldName(filter, newDatasets, dataId, newName, vi, {
mergeDomain: vi > 0
});
if (!updated) {
return null;
}
+ const leavingTimeFilter =
+ filter.type === FILTER_TYPES.timeRange && updated.type !== FILTER_TYPES.timeRange;
+
+ const oldEndName = toArray((filter as TimeRangeFilter).endName)[vi];
+ if (updated.type === FILTER_TYPES.timeRange && oldEndName) {
+ const endName = renames[oldEndName] ?? oldEndName;
+ updated =
+ applyTimeFilterEndFieldName(updated, newDatasets, dataId, endName, vi).filter ??
+ applyTimeFilterEndFieldName(updated, newDatasets, dataId, null, vi).filter ??
+ updated;
+ }
- // brush filters carry a second y-axis field; follow a rename there too
- const yAxis =
- updated.yAxis?.name &&
- (renames[updated.yAxis.name]
- ? {...updated.yAxis, name: renames[updated.yAxis.name]}
- : newFieldsByName.has(updated.yAxis.name)
- ? updated.yAxis
- : null);
+ // Time plots only accept numeric y-axis fields. Re-resolve the complete
+ // field so type and fieldIdx also follow a schema change.
+ let yAxis = updated.yAxis;
+ if (vi === 0 && updated.yAxis?.name) {
+ const yAxisName = renames[updated.yAxis.name] ?? updated.yAxis.name;
+ const yAxisField = newFieldsByName.get(yAxisName);
+ yAxis = yAxisField && FILTER_PLOT_Y_AXIS_TYPES.has(yAxisField.type) ? yAxisField : null;
+ }
- return {
+ let nextFilter = {
...updated,
yAxis,
+ ...(updated.yAxis && !yAxis
+ ? {plotType: {...updated.plotType, type: PLOT_TYPES.histogram}}
+ : {}),
// clamp the user's previous selection into the new domain instead of
// resetting it to the full range
value: adjustValueToFilterDomain(filter.value, updated)
};
+ if (leavingTimeFilter) {
+ nextFilter = clearTimelineFilterState(nextFilter);
+ } else if (vi === 0 && nextFilter.type === FILTER_TYPES.timeRange) {
+ const groupBy = nextFilter.plotType.groupBy;
+ if (groupBy && renames[groupBy.fieldName]) {
+ nextFilter.plotType = {
+ ...nextFilter.plotType,
+ groupBy: {...groupBy, fieldName: renames[groupBy.fieldName]}
+ };
+ }
+ // Timeline bins can remain identical while y-axis or group-by values
+ // change, so do not reuse the old series cache after a data rebuild.
+ nextFilter = {...nextFilter, lineChart: undefined};
+ }
+ // Keep existing plots; time plots are rebuilt after filtering below.
+ return filter.type === nextFilter.type || nextFilter.type === FILTER_TYPES.timeRange
+ ? nextFilter
+ : updateFilterPlot(newDatasets, nextFilter, dataId, state.layers);
})
.filter(Boolean) as Filter[];
- // reassign GPU channels now that the filter list is settled
- newFilters = newFilters.map(filter =>
- filter.gpu ? assignGpuChannel(setFilterGpuMode(filter, newFilters), newFilters) : filter
- );
-
- // 3. rebuild the table's filter state (gpuFilter / filterRecord / filteredIndex)
- newTable.filterTable(
- newFilters,
- state.layers.filter(l => l.config.dataId === dataId),
- {}
- );
-
- // 4. reconcile layers
+ // 3. reconcile layers
const layersToRemove: string[] = [];
const newLayers = state.layers.map(layer => {
if (layer.config.dataId !== dataId) {
@@ -4560,6 +4654,38 @@ export function updateDatasetUpdater(
return layer; // placeholder, dropped via removeLayerUpdater below
});
+ // Polygon filters reference layers, not fields. Preserve them for schema
+ // edits that keep their target layers, but remove invalid targets before
+ // rebuilding filter records when a required layer column was deleted.
+ const removedLayerIds = new Set(layersToRemove);
+ if (removedLayerIds.size) {
+ newFilters = newFilters.flatMap(filter => {
+ if (
+ filter.type !== FILTER_TYPES.polygon ||
+ !filter.layerId?.some(id => removedLayerIds.has(id))
+ ) {
+ return [filter];
+ }
+ filter.dataId.forEach(id => datasetIdsToRefilter.add(id));
+ const layerId = filter.layerId.filter(id => !removedLayerIds.has(id));
+ return layerId.length ? [{...filter, layerId}] : [];
+ });
+ }
+
+ // Reassign GPU channels now that the filter list is settled.
+ newFilters = newFilters.map(filter =>
+ filter.gpu ? assignGpuChannel(setFilterGpuMode(filter, newFilters), newFilters) : filter
+ );
+
+ // 4. rebuild the table's filter state using only layers that will survive
+ const survivingLayers = newLayers.filter(layer => !removedLayerIds.has(layer.id));
+ const filteredDatasets = applyFiltersToDatasets(
+ Array.from(datasetIdsToRefilter),
+ newDatasets,
+ newFilters,
+ survivingLayers
+ );
+
// 5. reconcile tooltip fieldsToShow
let interactionConfig = state.interactionConfig;
const tooltip = interactionConfig?.tooltip;
@@ -4583,7 +4709,7 @@ export function updateDatasetUpdater(
let nextState = {
...state,
- datasets: newDatasets,
+ datasets: filteredDatasets,
filters: newFilters,
layers: newLayers,
interactionConfig,
@@ -4593,6 +4719,10 @@ export function updateDatasetUpdater(
// 6. drop layers that lost their required columns
nextState = layersToRemove.reduce((accu, id) => removeLayerUpdater(accu, {id}), nextState);
+ // Filtering and polygon-layer removal can change the rows behind timeline
+ // plots, so rebuild their cached bins/series from the reconciled state.
+ nextState = updateFilterPlotsAfterFiltering(nextState, Array.from(datasetIdsToRefilter));
+
// 7. recompute layer domains + data for the updated dataset
nextState = updateAllLayerDomainData(nextState, dataId);
@@ -4623,6 +4753,45 @@ export function renameTableColumnUpdater(
});
}
+/**
+ * Convert a complete column before committing its new type. A failed value
+ * leaves the table and every reference unchanged.
+ */
+export function changeTableColumnTypeUpdater(
+ state: VisState,
+ action: VisStateActions.ChangeTableColumnTypeUpdaterAction
+): VisState {
+ const {dataId, fieldName, newType} = action;
+ const dataset = state.datasets[dataId];
+ if (!dataset || !canMutateDatasetSchema(dataset)) {
+ return state;
+ }
+ const result = buildChangeColumnTypeData(dataset, fieldName, newType);
+ if (!result) {
+ return state;
+ }
+ if (result.error) {
+ return withTask(
+ state,
+ ACTION_TASK_ADD_NOTIFICATION().map(() =>
+ addNotification(
+ errorNotification({
+ id: `column-type-${dataId}-${fieldName}`,
+ message: `Cannot change “${fieldName}” to ${newType}: ${formatColumnTypeConversionError(
+ result.error
+ )}. No values were changed.`
+ })
+ )
+ )
+ );
+ }
+ return updateDatasetUpdater(state, {
+ dataId,
+ data: result.payload.data,
+ renames: result.payload.renames
+ });
+}
+
/**
* Drop a column and the layers, filters, and other references that required it.
* The last remaining column is kept.
@@ -5930,8 +6099,14 @@ export function loadColumnStatsUpdater(
const tasks = fieldsToLoad.map(f =>
LOAD_COLUMN_STATS_TASK({dataset: newDataset, field: f}).bimap(
- ({result, filterProps}) => loadColumnStatsSuccess(newDataset.id, f.name, result, filterProps),
- (error: Error) => loadColumnStatsError(newDataset.id, f.name, error)
+ ({result, filterProps}) =>
+ loadColumnStatsSuccess(newDataset.id, f.name, result, filterProps, {
+ fieldType: f.type
+ }),
+ (error: Error) =>
+ loadColumnStatsError(newDataset.id, f.name, error, {
+ fieldType: f.type
+ })
)
);
const nextState = pick_('datasets')(merge_({[dataId]: newDataset}))(state);
@@ -5946,11 +6121,17 @@ export function loadColumnStatsUpdater(
*/
export function loadColumnStatsSuccessUpdater(
state: VisState,
- {result, filterProps, dataId, fieldName}: VisStateActions.LoadColumnStatsSuccessUpdaterAction
+ {
+ result,
+ filterProps,
+ dataId,
+ fieldName,
+ fieldType
+ }: VisStateActions.LoadColumnStatsSuccessUpdaterAction
): VisState {
const dataset = state.datasets[dataId];
const field = (dataset?.fields || []).find(f => f.name === fieldName);
- if (!dataset || !field) {
+ if (!dataset || !field || (fieldType !== undefined && field.type !== fieldType)) {
return state;
}
@@ -5972,14 +6153,14 @@ export function loadColumnStatsSuccessUpdater(
*/
export function loadColumnStatsErrorUpdater(
state: VisState,
- {error, dataId, fieldName}: VisStateActions.LoadColumnStatsErrorUpdaterAction
+ {error, dataId, fieldName, fieldType}: VisStateActions.LoadColumnStatsErrorUpdaterAction
): VisState {
- Console.error(error);
const dataset = state.datasets[dataId];
const field = (dataset?.fields || []).find(f => f.name === fieldName);
- if (!dataset || !field) {
+ if (!dataset || !field || (fieldType !== undefined && field.type !== fieldType)) {
return state;
}
+ Console.error(error);
const newFields = dataset.fields.map(f =>
f.name === fieldName ? {...f, isLoadingStats: false} : f
);
diff --git a/src/reducers/src/vis-state.ts b/src/reducers/src/vis-state.ts
index 027e18fbf..f016ab5c1 100644
--- a/src/reducers/src/vis-state.ts
+++ b/src/reducers/src/vis-state.ts
@@ -147,6 +147,8 @@ const actionHandler = {
[ActionTypes.RENAME_TABLE_COLUMN]: visStateUpdaters.renameTableColumnUpdater,
+ [ActionTypes.CHANGE_TABLE_COLUMN_TYPE]: visStateUpdaters.changeTableColumnTypeUpdater,
+
[ActionTypes.DELETE_TABLE_COLUMN]: visStateUpdaters.deleteTableColumnUpdater,
[ActionTypes.SET_FEATURES]: visStateUpdaters.setFeaturesUpdater,
diff --git a/src/table/src/dataset-ops/column-edit.spec.ts b/src/table/src/dataset-ops/column-edit.spec.ts
index 9c8dfdaf0..6e15ed6b5 100644
--- a/src/table/src/dataset-ops/column-edit.spec.ts
+++ b/src/table/src/dataset-ops/column-edit.spec.ts
@@ -2,8 +2,15 @@
// Copyright contributors to the kepler.gl project
import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
+import {createArrowVector} from '@kepler.gl/utils';
import KeplerTable from '../kepler-table';
-import {buildDeleteColumnData, buildRenameColumnData} from './column-edit';
+import {
+ buildChangeColumnTypeData,
+ buildDeleteColumnData,
+ buildRenameColumnData,
+ formatColumnTypeConversionError,
+ getCompatibleColumnTypes
+} from './column-edit';
function makeTable() {
const table = new KeplerTable({
@@ -23,6 +30,22 @@ function makeTable() {
return table;
}
+function makeColumnTable(
+ values: unknown[],
+ type: string = ALL_FIELD_TYPES.string,
+ analyzerType?: string
+) {
+ const table = new KeplerTable({
+ info: {id: 'typed', label: 'typed', type: 'local'},
+ color: [0, 0, 0]
+ });
+ table.updateSchema({
+ fields: [{name: 'value', type, analyzerType}],
+ rows: values.map(value => [value])
+ });
+ return table;
+}
+
describe('column edit payloads', () => {
test('rename keeps values and records the old name', () => {
const payload = buildRenameColumnData(makeTable(), 'region', 'area');
@@ -82,4 +105,158 @@ describe('column edit payloads', () => {
});
expect(buildDeleteColumnData(oneColumn, 'region')).toBeNull();
});
+
+ test('type change converts every value and updates field metadata', () => {
+ const table = makeColumnTable(['10', '-2', null], ALL_FIELD_TYPES.string, 'STRING');
+ table.fields[0].displayFormat = '.2f';
+
+ const result = buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.integer);
+ expect(result?.payload?.data.rows).toEqual([[10], [-2], [null]]);
+ expect(result?.payload?.data.fields[0]).toMatchObject({
+ name: 'value',
+ type: ALL_FIELD_TYPES.integer,
+ analyzerType: 'INT',
+ format: ''
+ });
+ expect(
+ (result?.payload?.data.fields[0] as {displayFormat?: string} | undefined)?.displayFormat
+ ).toBeUndefined();
+ });
+
+ test('type change reports the first invalid row without mutating the table', () => {
+ const table = makeColumnTable(['10', 'not-a-number', '30'], ALL_FIELD_TYPES.string, 'STRING');
+
+ const result = buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.integer);
+ expect(result).toEqual({
+ error: {rowIndex: 1, value: 'not-a-number', reason: 'is not a safe integer'}
+ });
+ expect(table.fields[0].type).toBe(ALL_FIELD_TYPES.string);
+ expect(table.dataContainer.rowAsArray(0)[0]).toBe('10');
+ expect(getCompatibleColumnTypes(table.dataContainer, 0, ALL_FIELD_TYPES.string)).toEqual([
+ ALL_FIELD_TYPES.string
+ ]);
+ expect(result?.error && formatColumnTypeConversionError(result.error)).toBe(
+ 'row 2 value “not-a-number” is not a safe integer'
+ );
+ });
+
+ test('type change parses timestamps and rejects types outside the initial scope', () => {
+ const table = makeColumnTable(
+ ['2024-01-01T12:30:00Z', '2025-02-02T01:02:03Z'],
+ ALL_FIELD_TYPES.string,
+ 'STRING'
+ );
+
+ const timestamp = buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.timestamp);
+ expect(timestamp?.payload?.data.fields[0]).toMatchObject({
+ type: ALL_FIELD_TYPES.timestamp,
+ analyzerType: 'DATETIME',
+ format: 'YYYY-M-DTH:m:s'
+ });
+
+ expect(buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.boolean)).toBeNull();
+ expect(buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.array)).toBeNull();
+ });
+
+ test.each([
+ {type: ALL_FIELD_TYPES.integer, values: [1_700_000_000, 1_700_000_001]},
+ {type: ALL_FIELD_TYPES.real, values: [1_700_000_000.4, 1_700_000_001.6]},
+ {type: ALL_FIELD_TYPES.string, values: ['1700000000.4', '1700000001.6']}
+ ])('timestamp conversion preserves plausible $type epochs without rounding', ({type, values}) => {
+ const table = makeColumnTable(values, type);
+ const converted = buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.timestamp);
+ expect(converted?.payload?.data.rows).toEqual(
+ type === ALL_FIELD_TYPES.integer
+ ? [[1_700_000_000], [1_700_000_001]]
+ : [[1_700_000_000.4], [1_700_000_001.6]]
+ );
+ expect(converted?.payload?.data.fields[0]).toMatchObject({
+ type: ALL_FIELD_TYPES.timestamp,
+ analyzerType: 'TIME',
+ format: 'x'
+ });
+ expect(getCompatibleColumnTypes(table.dataContainer, 0, type)).toContain(
+ ALL_FIELD_TYPES.timestamp
+ );
+ });
+
+ test('timestamp conversion rejects measurements', () => {
+ const measurements = makeColumnTable([3.1, 4], ALL_FIELD_TYPES.real);
+ expect(buildChangeColumnTypeData(measurements, 'value', ALL_FIELD_TYPES.timestamp)).toEqual({
+ error: {
+ rowIndex: 0,
+ value: 3.1,
+ reason: 'is not a plausible epoch timestamp'
+ }
+ });
+ expect(getCompatibleColumnTypes(measurements.dataContainer, 0, ALL_FIELD_TYPES.real)).toEqual([
+ ALL_FIELD_TYPES.real,
+ ALL_FIELD_TYPES.string
+ ]);
+ });
+
+ test('type changes omit numeric conversions that do not change column behavior', () => {
+ const integers = makeColumnTable([1_700_000_000, 1_700_000_001], ALL_FIELD_TYPES.integer);
+ expect(getCompatibleColumnTypes(integers.dataContainer, 0, ALL_FIELD_TYPES.integer)).toEqual([
+ ALL_FIELD_TYPES.integer,
+ ALL_FIELD_TYPES.string,
+ ALL_FIELD_TYPES.timestamp
+ ]);
+ expect(buildChangeColumnTypeData(integers, 'value', ALL_FIELD_TYPES.real)).toBeNull();
+
+ const timestamp = buildChangeColumnTypeData(integers, 'value', ALL_FIELD_TYPES.timestamp);
+ expect(timestamp?.payload).toBeTruthy();
+ if (!timestamp?.payload) {
+ throw new Error('Expected timestamp conversion to succeed');
+ }
+ const timestamps = makeColumnTable([]);
+ timestamps.updateSchema(timestamp.payload.data);
+ expect(
+ getCompatibleColumnTypes(timestamps.dataContainer, 0, ALL_FIELD_TYPES.timestamp)
+ ).toEqual([ALL_FIELD_TYPES.integer, ALL_FIELD_TYPES.string, ALL_FIELD_TYPES.timestamp]);
+ expect(buildChangeColumnTypeData(timestamps, 'value', ALL_FIELD_TYPES.real)).toBeNull();
+ });
+
+ test('real conversion does not silently round large integer strings', () => {
+ for (const value of [
+ '9007199254740993',
+ '9007199254740993.0',
+ '9007199254740993e0',
+ '9.007199254740993e15'
+ ]) {
+ const table = makeColumnTable([value]);
+
+ expect(buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.real)).toEqual({
+ error: {
+ rowIndex: 0,
+ value,
+ reason: 'loses integer precision as a real number'
+ }
+ });
+ expect(getCompatibleColumnTypes(table.dataContainer, 0, ALL_FIELD_TYPES.string)).toEqual([
+ ALL_FIELD_TYPES.string
+ ]);
+ }
+ });
+
+ test('type change preserves untouched Arrow vectors', () => {
+ const table = new KeplerTable({
+ info: {id: 'arrow', label: 'arrow', type: 'local'},
+ color: [0, 0, 0]
+ });
+ table.updateSchema({
+ fields: [
+ {name: 'value', type: ALL_FIELD_TYPES.string},
+ {name: 'label', type: ALL_FIELD_TYPES.string}
+ ],
+ rows: [],
+ cols: [createArrowVector(['10', '20']), createArrowVector(['a', 'b'])]
+ });
+ const untouched = table.dataContainer.getColumn?.(1);
+
+ const result = buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.integer);
+ expect(result?.payload?.data.rows).toEqual([]);
+ expect(result?.payload?.data.cols?.[1]).toBe(untouched);
+ expect(Array.from(result?.payload?.data.cols?.[0] as Iterable<unknown>)).toEqual([10, 20]);
+ });
});
diff --git a/src/table/src/dataset-ops/column-edit.ts b/src/table/src/dataset-ops/column-edit.ts
index facfb6e2c..287d9e1ef 100644
--- a/src/table/src/dataset-ops/column-edit.ts
+++ b/src/table/src/dataset-ops/column-edit.ts
@@ -1,8 +1,10 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
+import {Analyzer, DATA_TYPES as AnalyzerDataTypes} from 'type-analyzer';
+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
import {Field, ProtoDataset} from '@kepler.gl/types';
-import {DataContainerInterface} from '@kepler.gl/utils';
+import {createArrowVector, DataContainerInterface} from '@kepler.gl/utils';
export type ColumnEditDataset = {
fields: Field[];
@@ -14,6 +16,279 @@ export type ColumnEditPayload = {
renames: Record<string, string>;
};
+export const EDITABLE_FIELD_TYPES = [
+ ALL_FIELD_TYPES.integer,
+ ALL_FIELD_TYPES.real,
+ ALL_FIELD_TYPES.string,
+ ALL_FIELD_TYPES.timestamp
+] as const;
+
+export type EditableFieldType = (typeof EDITABLE_FIELD_TYPES)[number];
+
+const EDITABLE_TARGET_TYPES: Record<EditableFieldType, readonly EditableFieldType[]> = {
+ [ALL_FIELD_TYPES.integer]: [ALL_FIELD_TYPES.string, ALL_FIELD_TYPES.timestamp],
+ [ALL_FIELD_TYPES.real]: [ALL_FIELD_TYPES.string, ALL_FIELD_TYPES.timestamp],
+ [ALL_FIELD_TYPES.string]: [
+ ALL_FIELD_TYPES.integer,
+ ALL_FIELD_TYPES.real,
+ ALL_FIELD_TYPES.timestamp
+ ],
+ [ALL_FIELD_TYPES.timestamp]: [ALL_FIELD_TYPES.integer, ALL_FIELD_TYPES.string]
+};
+
+function isEditableFieldType(type: string): type is EditableFieldType {
+ return (EDITABLE_FIELD_TYPES as readonly string[]).includes(type);
+}
+
+export type ColumnTypeConversionError = {
+ rowIndex: number;
+ value: unknown;
+ reason: string;
+};
+
+export type ChangeColumnTypeResult =
+ | {payload: ColumnEditPayload; error?: never}
+ | {payload?: never; error: ColumnTypeConversionError};
+
+function columnValuePreview(value: unknown): string {
+ let preview: string;
+ try {
+ preview = typeof value === 'string' ? value : JSON.stringify(value);
+ } catch {
+ preview = String(value);
+ }
+ if (preview === undefined) {
+ preview = String(value);
+ }
+ return preview.length > 80 ? `${preview.slice(0, 77)}...` : preview;
+}
+
+export function formatColumnTypeConversionError(error: ColumnTypeConversionError): string {
+ return `row ${error.rowIndex + 1} value “${columnValuePreview(error.value)}” ${error.reason}`;
+}
+
+const DECIMAL_NUMBER = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/i;
+const INTEGER = /^[+-]?\d+$/;
+const MIN_ABSOLUTE_EPOCH = 100_000_000;
+const MILLISECOND_EPOCH_THRESHOLD = 100_000_000_000;
+const MAX_ABSOLUTE_EPOCH_MILLISECONDS = 9_999_999_999_999;
+
+const ANALYZER_TYPE_BY_FIELD_TYPE: Record<string, string> = {
+ [ALL_FIELD_TYPES.integer]: AnalyzerDataTypes.INT,
+ [ALL_FIELD_TYPES.real]: AnalyzerDataTypes.FLOAT,
+ [ALL_FIELD_TYPES.string]: AnalyzerDataTypes.STRING,
+ [ALL_FIELD_TYPES.timestamp]: AnalyzerDataTypes.DATETIME
+};
+
+function analyzerMetadata(value: unknown): {type: string; format: string} | null {
+ const metadata = Analyzer.computeColMeta([{value}])?.[0];
+ return metadata ? {type: metadata.type, format: metadata.format || ''} : null;
+}
+
+type ColumnConversionState = {
+ analyzerType: string;
+ format: string;
+};
+
+function conversionState(type: EditableFieldType): ColumnConversionState {
+ return {
+ analyzerType: type === ALL_FIELD_TYPES.timestamp ? '' : ANALYZER_TYPE_BY_FIELD_TYPE[type],
+ format: ''
+ };
+}
+
+function numericValue(value: unknown, integerOnly: boolean): number {
+ if (typeof value === 'number') {
+ return value;
+ }
+ if (typeof value === 'bigint') {
+ return Number(value);
+ }
+ if (typeof value !== 'string') {
+ return Number.NaN;
+ }
+ const trimmed = value.trim();
+ if (!(integerOnly ? INTEGER : DECIMAL_NUMBER).test(trimmed)) {
+ return Number.NaN;
+ }
+ return Number(trimmed);
+}
+
+function exactIntegerFromDecimal(value: string): bigint | null {
+ const trimmed = value.trim().toLowerCase();
+ if (!DECIMAL_NUMBER.test(trimmed)) {
+ return null;
+ }
+
+ const unsigned = trimmed.replace(/^[+-]/, '');
+ const [mantissa, exponentString] = unsigned.split('e');
+ const [whole, fraction = ''] = mantissa.split('.');
+ const digits = `${whole || '0'}${fraction}`;
+ if (BigInt(digits) === 0n) {
+ return 0n;
+ }
+ const exponent = exponentString ? Number(exponentString) : 0;
+ if (!Number.isSafeInteger(exponent)) {
+ return null;
+ }
+
+ const decimalPlaces = fraction.length - exponent;
+ let integerDigits: string;
+ if (decimalPlaces <= 0) {
+ integerDigits = `${digits}${'0'.repeat(-decimalPlaces)}`;
+ } else {
+ const fractionalTail = digits.slice(-decimalPlaces);
+ if (decimalPlaces >= digits.length || /[1-9]/.test(fractionalTail)) {
+ return null;
+ }
+ integerDigits = digits.slice(0, -decimalPlaces);
+ }
+ const integer = BigInt(integerDigits);
+ return trimmed.startsWith('-') ? -integer : integer;
+}
+
+function losesIntegerPrecision(value: unknown, parsed: number): boolean {
+ if (!Number.isFinite(parsed) || !Number.isInteger(parsed)) {
+ return false;
+ }
+ if (typeof value === 'bigint') {
+ return BigInt(parsed) !== value;
+ }
+ if (typeof value === 'string') {
+ const exactInteger = exactIntegerFromDecimal(value);
+ return exactInteger !== null && BigInt(parsed) !== exactInteger;
+ }
+ return false;
+}
+
+function convertColumnValue(
+ value: unknown,
+ type: EditableFieldType,
+ rowIndex: number,
+ state: ColumnConversionState,
+ output?: unknown[]
+): ColumnTypeConversionError | null {
+ if (value === null || value === undefined) {
+ output?.push(null);
+ return null;
+ }
+
+ switch (type) {
+ case ALL_FIELD_TYPES.string: {
+ try {
+ const serialized =
+ value instanceof Date
+ ? value.toISOString()
+ : typeof value === 'object'
+ ? JSON.stringify(value)
+ : String(value);
+ output?.push(serialized ?? String(value));
+ return null;
+ } catch {
+ return {rowIndex, value, reason: 'cannot be serialized as a string'};
+ }
+ }
+ case ALL_FIELD_TYPES.integer:
+ case ALL_FIELD_TYPES.real: {
+ const integerOnly = type === ALL_FIELD_TYPES.integer;
+ const parsed = numericValue(value, integerOnly);
+ if (integerOnly ? !Number.isSafeInteger(parsed) : !Number.isFinite(parsed)) {
+ return {
+ rowIndex,
+ value,
+ reason: integerOnly ? 'is not a safe integer' : 'is not a finite number'
+ };
+ }
+ if (!integerOnly && losesIntegerPrecision(value, parsed)) {
+ return {rowIndex, value, reason: 'loses integer precision as a real number'};
+ }
+ output?.push(parsed);
+ return null;
+ }
+ case ALL_FIELD_TYPES.timestamp: {
+ const source = value instanceof Date ? value.toISOString() : value;
+ const isNumeric =
+ typeof source === 'number' ||
+ typeof source === 'bigint' ||
+ (typeof source === 'string' && DECIMAL_NUMBER.test(source.trim()));
+ if (isNumeric) {
+ const parsed = numericValue(source, false);
+ const numericFormat = Math.abs(parsed) >= MILLISECOND_EPOCH_THRESHOLD ? 'X' : 'x';
+ const milliseconds = numericFormat === 'x' ? parsed * 1000 : parsed;
+ if (
+ !Number.isFinite(parsed) ||
+ Math.abs(parsed) < MIN_ABSOLUTE_EPOCH ||
+ Math.abs(milliseconds) > MAX_ABSOLUTE_EPOCH_MILLISECONDS
+ ) {
+ return {rowIndex, value, reason: 'is not a plausible epoch timestamp'};
+ }
+ if (state.format && state.format !== numericFormat) {
+ return {rowIndex, value, reason: 'uses a different timestamp format'};
+ }
+ state.analyzerType = AnalyzerDataTypes.TIME;
+ state.format = numericFormat;
+ output?.push(parsed);
+ return null;
+ }
+
+ if (typeof source !== 'string') {
+ return {rowIndex, value, reason: 'is not a timestamp'};
+ }
+ const metadata = analyzerMetadata(source);
+ if (
+ !metadata ||
+ ![AnalyzerDataTypes.TIME, AnalyzerDataTypes.DATETIME].includes(metadata.type)
+ ) {
+ return {rowIndex, value, reason: 'is not a valid timestamp'};
+ }
+ if (
+ (state.format && state.format !== metadata.format) ||
+ (state.analyzerType && state.analyzerType !== metadata.type)
+ ) {
+ return {rowIndex, value, reason: 'uses a different timestamp format'};
+ }
+ state.analyzerType = metadata.type;
+ state.format = metadata.format;
+ output?.push(source);
+ return null;
+ }
+ default:
+ return {rowIndex, value, reason: `cannot be converted to ${type}`};
+ }
+}
+
+/** Validate every candidate in one pass without allocating converted columns. */
+export function getCompatibleColumnTypes(
+ dataContainer: DataContainerInterface,
+ fieldIndex: number,
+ currentType: string
+): EditableFieldType[] {
+ if (
+ fieldIndex < 0 ||
+ fieldIndex >= dataContainer.numColumns() ||
+ !isEditableFieldType(currentType)
+ ) {
+ return [];
+ }
+
+ const candidates = new Map<EditableFieldType, ColumnConversionState>();
+ for (const type of EDITABLE_TARGET_TYPES[currentType]) {
+ candidates.set(type, conversionState(type));
+ }
+
+ const rowCount = dataContainer.numRows();
+ for (let rowIndex = 0; rowIndex < rowCount && candidates.size; rowIndex++) {
+ const value = dataContainer.valueAt(rowIndex, fieldIndex);
+ for (const [type, state] of candidates) {
+ if (convertColumnValue(value, type, rowIndex, state)) {
+ candidates.delete(type);
+ }
+ }
+ }
+
+ return EDITABLE_FIELD_TYPES.filter(type => type === currentType || candidates.has(type));
+}
+
/**
* Field descriptor for `updateDataset`, with accessors and per-column stats
* stripped so the table rebuilds them against the new container.
@@ -107,3 +382,76 @@ export function buildDeleteColumnData(
renames: {}
};
}
+
+/**
+ * Convert an entire column before returning an update payload. A failed value
+ * returns its row and leaves the source dataset untouched.
+ */
+export function buildChangeColumnTypeData(
+ dataset: ColumnEditDataset,
+ fieldName: string,
+ newType: string
+): ChangeColumnTypeResult | null {
+ if (!isEditableFieldType(newType)) {
+ return null;
+ }
+ const fieldIndex = dataset.fields.findIndex(field => field.name === fieldName);
+ const currentType = dataset.fields[fieldIndex]?.type;
+ if (
+ fieldIndex < 0 ||
+ !isEditableFieldType(currentType) ||
+ !EDITABLE_TARGET_TYPES[currentType].includes(newType)
+ ) {
+ return null;
+ }
+
+ const state = conversionState(newType);
+ const values: unknown[] = [];
+ const rowCount = dataset.dataContainer.numRows();
+ for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
+ const error = convertColumnValue(
+ dataset.dataContainer.valueAt(rowIndex, fieldIndex),
+ newType,
+ rowIndex,
+ state,
+ values
+ );
+ if (error) {
+ return {error};
+ }
+ }
+
+ const fields = dataset.fields.map((field, index) => {
+ if (index !== fieldIndex) {
+ return schemaField(field, field.name);
+ }
+ const {
+ displayFormat: _displayFormat,
+ indexBy: _indexBy,
+ ...fieldWithoutTypeSpecificMetadata
+ } = field;
+ return {
+ ...schemaField(fieldWithoutTypeSpecificMetadata, field.name),
+ type: newType,
+ analyzerType: state.analyzerType || ANALYZER_TYPE_BY_FIELD_TYPE[newType],
+ format: state.format
+ };
+ });
+
+ if (typeof dataset.dataContainer.getColumn === 'function') {
+ const cols = dataset.fields.map((_, index) => dataset.dataContainer.getColumn?.(index));
+ if (cols.every(Boolean)) {
+ cols[fieldIndex] = createArrowVector(values);
+ return {payload: {data: {fields, rows: [], cols}, renames: {}}};
+ }
+ }
+
+ const rows: unknown[][] = new Array(rowCount);
+ for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
+ const row = Array.from(dataset.dataContainer.rowAsArray(rowIndex));
+ row[fieldIndex] = values[rowIndex];
+ rows[rowIndex] = row;
+ }
+
+ return {payload: {data: {fields, rows}, renames: {}}};
+}
diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.ts
index 245cc732e..6de64ebdd 100644
--- a/src/utils/src/arrow-data-container.ts
+++ b/src/utils/src/arrow-data-container.ts
@@ -62,6 +62,11 @@ export function isArrowVector(data: any): data is arrow.Vector {
);
}
+/** Build an Arrow vector while letting Arrow infer the physical type from converted values. */
+export function createArrowVector(values: unknown[]): arrow.Vector {
+ return arrow.vectorFromArray(values as any[]);
+}
+
/**
* Check if data is an Arrow FixedSizeList DataType.
* Uses duck-typing instead of `instanceof` to handle DuckDB's bundled Arrow version.
diff --git a/src/utils/src/index.ts b/src/utils/src/index.ts
index be976faf6..62fa317e8 100644
--- a/src/utils/src/index.ts
+++ b/src/utils/src/index.ts
@@ -158,6 +158,7 @@ export * from './map-utils';
export {
ArrowDataContainer,
+ createArrowVector,
compactArrowTable,
arrowDataTypeToAnalyzerDataType,
arrowDataTypeToFieldType,