Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
feat(data): allow changing column data types (#3834)
Igor Dykhta0ba38a30e010
diff --git a/src/actions/src/action-types.ts b/src/actions/src/action-types.tsindex 8d9e3d416..8f2ccef12 100644--- a/src/actions/src/action-types.ts+++ b/src/actions/src/action-types.ts@@ -82,6 +82,7 @@ export const ActionTypes = {UPDATE_DATASET_PROPS: `${ACTION_PREFIX}UPDATE_DATASET_PROPS`,UPDATE_DATASET: `${ACTION_PREFIX}UPDATE_DATASET`,RENAME_TABLE_COLUMN: `${ACTION_PREFIX}RENAME_TABLE_COLUMN`,+ CHANGE_TABLE_COLUMN_TYPE: `${ACTION_PREFIX}CHANGE_TABLE_COLUMN_TYPE`,DELETE_TABLE_COLUMN: `${ACTION_PREFIX}DELETE_TABLE_COLUMN`,TOGGLE_FILTER_ANIMATION: `${ACTION_PREFIX}TOGGLE_FILTER_ANIMATION`,UPDATE_FILTER_ANIMATION_SPEED: `${ACTION_PREFIX}UPDATE_FILTER_ANIMATION_SPEED`,diff --git a/src/actions/src/vis-state-actions.ts b/src/actions/src/vis-state-actions.tsindex ec68c1d64..c0870b0cc 100644--- a/src/actions/src/vis-state-actions.ts+++ b/src/actions/src/vis-state-actions.ts@@ -1372,6 +1372,7 @@ export type LoadColumnStatsSuccessUpdaterAction = {fieldName: string;result: any;filterProps?: any;+ fieldType?: string;};/**@@ -1383,7 +1384,8 @@ export function loadColumnStatsSuccess(dataId: string,fieldName: string,result: any,- filterProps?: any+ filterProps?: any,+ request?: {fieldType: string}): Merge<LoadColumnStatsSuccessUpdaterAction,{type: typeof ActionTypes.LOAD_COLUMN_STATS_SUCCESS}@@ -1393,7 +1395,8 @@ export function loadColumnStatsSuccess(dataId,fieldName,result,- filterProps+ filterProps,+ ...request};}@@ -1401,6 +1404,7 @@ export type LoadColumnStatsErrorUpdaterAction = {dataId: string;fieldName: string;error: Error;+ fieldType?: string;};/**@@ -1411,13 +1415,15 @@ export type LoadColumnStatsErrorUpdaterAction = {export function loadColumnStatsError(dataId: string,fieldName: string,- error: Error+ error: Error,+ request?: {fieldType: string}): Merge<LoadColumnStatsErrorUpdaterAction, {type: typeof ActionTypes.LOAD_COLUMN_STATS_ERROR}> {return {type: ActionTypes.LOAD_COLUMN_STATS_ERROR,dataId,fieldName,- error+ error,+ ...request};}@@ -1582,6 +1588,33 @@ export function renameTableColumn(};}+export type ChangeTableColumnTypeUpdaterAction = {+ dataId: string;+ fieldName: string;+ newType: string;+};+/**+ * Convert every value in a dataset column and update its field type atomically.+ * Invalid values leave the dataset unchanged and produce an error notification.+ * @param dataId - dataset id+ * @param fieldName - column name+ * @param newType - target kepler.gl field type+ * @returns action+ * @public+ */+export function changeTableColumnType(+ dataId: string,+ fieldName: string,+ newType: string+): Merge<ChangeTableColumnTypeUpdaterAction, {type: typeof ActionTypes.CHANGE_TABLE_COLUMN_TYPE}> {+ return {+ type: ActionTypes.CHANGE_TABLE_COLUMN_TYPE,+ dataId,+ fieldName,+ newType+ };+}+export type DeleteTableColumnUpdaterAction = {dataId: string;fieldName: string;diff --git a/src/components/src/common/data-table/header-cell.spec.tsx b/src/components/src/common/data-table/header-cell.spec.tsxindex da2704bfc..dd594ea27 100644--- a/src/components/src/common/data-table/header-cell.spec.tsx+++ b/src/components/src/common/data-table/header-cell.spec.tsx@@ -37,27 +37,41 @@ const HeaderCell = HeaderCellFactory(const columns = ['region', 'pop'];const colMeta = {- region: {name: 'region', type: 'string'},- pop: {name: 'pop', type: 'integer'}+ region: {name: 'region', type: 'string', colIdx: 0},+ pop: {name: 'pop', type: 'integer', colIdx: 1}};function ColumnHeader({renameTableColumn,+ changeTableColumnType,deleteTableColumn,- createFilterFromColumn+ createFilterFromColumn,+ metadata = colMeta,+ typeChangeValues}: {renameTableColumn?: (column: string, newName: string) => void;+ changeTableColumnType?: (column: string, newType: string) => void;deleteTableColumn?: (column: string) => void;createFilterFromColumn?: (column: string) => void;+ metadata?: typeof colMeta;+ typeChangeValues?: unknown[];}) {const [moreOptionsColumn, setMoreOptionsColumn] = useState<string | null>(null);const props = {- colMeta,+ colMeta: metadata,+ typeChangeDataContainer: typeChangeValues+ ? {+ numRows: () => typeChangeValues.length,+ numColumns: () => 1,+ valueAt: (rowIndex: number) => typeChangeValues[rowIndex]+ }+ : undefined,sortColumn: {},pinTableColumn: jest.fn(),copyTableColumn: jest.fn(),sortTableColumn: jest.fn(),renameTableColumn,+ changeTableColumnType,deleteTableColumn,createFilterFromColumn} as unknown as HeaderCellProps['props'];@@ -178,4 +192,84 @@ describe('column header menu', () => {fireEvent.click(screen.getByText('Create Filter'));expect(createFilterFromColumn).toHaveBeenCalledWith('region');});++ test('changes the column data type from the column menu', () => {+ const changeTableColumnType = jest.fn();+ const {container} = renderWithTheme(+ <ColumnHeader changeTableColumnType={changeTableColumnType} renameTableColumn={jest.fn()} />+ );++ openMenu(container);+ fireEvent.click(screen.getByText('Change Data Type'));+ expect(screen.queryByText('Boolean')).toBeNull();+ expect(screen.queryByText('GeoJSON')).toBeNull();+ fireEvent.click(screen.getByText('Integer'));+ expect(changeTableColumnType).toHaveBeenCalledWith('region', 'integer');+ });++ test.each([false, true])(+ 'does not scan the existing menu (type changes enabled: %s)',+ enabled => {+ const readValue = jest.fn(() => '10');+ const values = [null];+ Object.defineProperty(values, 0, {get: readValue});+ const {container} = renderWithTheme(+ <ColumnHeader+ changeTableColumnType={enabled ? jest.fn() : undefined}+ typeChangeValues={values}+ />+ );++ openMenu(container);+ expect(readValue).not.toHaveBeenCalled();+ expect(screen.getByText('Copy Column')).toBeTruthy();+ if (enabled) {+ fireEvent.click(screen.getByText('Change Data Type'));+ expect(readValue).toHaveBeenCalled();+ } else {+ expect(screen.queryByText('Change Data Type')).toBeNull();+ }+ }+ );++ test.each([+ {type: 'string', values: ['ML', 'MW'], omitted: ['Integer', 'Real', 'Timestamp']},+ {type: 'real', values: [3.1, 4], omitted: ['Integer', 'Timestamp']},+ {type: 'real', values: [1_700_000_000.4, 1_700_000_001.6], omitted: ['Integer']},+ {type: 'integer', values: [1, 2], omitted: ['Real', 'Timestamp']}+ ])('offers compatible types for $type values $values', ({type, values, omitted}) => {+ const changeTableColumnType = jest.fn();+ const {container} = renderWithTheme(+ <ColumnHeader+ changeTableColumnType={changeTableColumnType}+ metadata={{...colMeta, region: {...colMeta.region, type}}}+ typeChangeValues={values}+ />+ );++ openMenu(container);+ fireEvent.click(screen.getByText('Change Data Type'));+ omitted.forEach(label => expect(screen.queryByText(label)).toBeNull());+ if (!omitted.includes('Timestamp')) expect(screen.getByText('Timestamp')).toBeTruthy();+ fireEvent.click(screen.getByText('String'));+ if (type === 'string') expect(changeTableColumnType).not.toHaveBeenCalled();+ else expect(changeTableColumnType).toHaveBeenCalledWith('region', 'string');+ });++ test('hides type changes for field types outside the initial scope', () => {+ const metadata = {+ ...colMeta,+ region: {...colMeta.region, type: 'boolean'}+ };+ const {container} = renderWithTheme(+ <ColumnHeader+ changeTableColumnType={jest.fn()}+ metadata={metadata}+ typeChangeValues={[true, false]}+ />+ );++ openMenu(container);+ expect(screen.queryByText('Change Data Type')).toBeNull();+ });});diff --git a/src/components/src/common/data-table/header-cell.tsx b/src/components/src/common/data-table/header-cell.tsxindex 0b4e14c07..d9d251911 100644--- a/src/components/src/common/data-table/header-cell.tsx+++ b/src/components/src/common/data-table/header-cell.tsx@@ -197,6 +197,8 @@ function HeaderCellFactory(const {columnIndex, key, style} = cellInfo;const {colMeta,+ dataContainer,+ typeChangeDataContainer,sortColumn,sortTableColumn,pinTableColumn,@@ -207,6 +209,7 @@ function HeaderCellFactory(dataId,loadColumnStats,renameTableColumn,+ changeTableColumnType,deleteTableColumn,createFilterFromColumn} = props;@@ -383,6 +386,9 @@ function HeaderCellFactory(copyTableColumn={onCopy}setDisplayFormat={setColumnDisplayFormat ? onSetDisplayFormat : undefined}renameTableColumn={renameTableColumn}+ changeTableColumnType={changeTableColumnType}+ typeChangeDataContainer={typeChangeDataContainer ?? dataContainer}+ columnIndex={colMeta[column]?.colIdx ?? columns.indexOf(column)}deleteTableColumn={deleteTableColumn}createFilterFromColumn={createFilterFromColumn}onStartRename={startRename}diff --git a/src/components/src/common/data-table/index.tsx b/src/components/src/common/data-table/index.tsxindex d7873d767..71117fcf4 100644--- a/src/components/src/common/data-table/index.tsx+++ b/src/components/src/common/data-table/index.tsx@@ -449,6 +449,8 @@ export interface DataTableProps {fixedWidth?: number | null;theme?: any;dataContainer: DataContainerInterface | null;+ /** Unfiltered source data used to validate whole-column type changes. */+ typeChangeDataContainer?: DataContainerInterface | null;fixedHeight?: number | null;colMeta: ColMeta;sortColumn: SortColumn;@@ -458,6 +460,7 @@ export interface DataTableProps {setColumnDisplayFormat?: (formats: {[key: string]: string}) => void;copyTableColumn: (column: string) => void;renameTableColumn?: (column: string, newName: string) => void;+ changeTableColumnType?: (column: string, newType: string) => void;deleteTableColumn?: (column: string) => void;createFilterFromColumn?: (column: string) => void;sortOrder?: number[] | null;diff --git a/src/components/src/common/data-table/option-dropdown.tsx b/src/components/src/common/data-table/option-dropdown.tsxindex b7d43216d..710807df2 100644--- a/src/components/src/common/data-table/option-dropdown.tsx+++ b/src/components/src/common/data-table/option-dropdown.tsx@@ -1,19 +1,21 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import React, {useCallback, useState} from 'react';+import React, {useCallback, useMemo, useState} from 'react';import styled from 'styled-components';import Portaled from '../portaled';import DropdownList from '../item-selector/dropdown-list';import {+ ALL_FIELD_TYPES,SORT_ORDER,TABLE_OPTION,TABLE_OPTION_LIST,TooltipFormat,TableOption} from '@kepler.gl/constants';-import {getFieldFormatLabels} from '@kepler.gl/utils';+import {DataContainerInterface, getFieldFormatLabels} from '@kepler.gl/utils';import {ColMeta} from '@kepler.gl/types';+import {getCompatibleColumnTypes} from '@kepler.gl/table';import {ArrowDown,ArrowUp,@@ -166,11 +168,23 @@ export interface OptionDropdownProps {isSorted?: string;isPinned?: boolean;renameTableColumn?: (column: string, newName: string) => void;+ changeTableColumnType?: (column: string, newType: string) => void;+ typeChangeDataContainer?: DataContainerInterface | null;+ columnIndex?: number;deleteTableColumn?: (column: string) => void;createFilterFromColumn?: (column: string) => void;onStartRename?: () => void;}+const COLUMN_TYPE_OPTIONS = [+ {type: ALL_FIELD_TYPES.integer, display: 'Integer'},+ {type: ALL_FIELD_TYPES.real, display: 'Real'},+ {type: ALL_FIELD_TYPES.string, display: 'String'},+ {type: ALL_FIELD_TYPES.timestamp, display: 'Timestamp'}+];++type ColumnTypeOption = (typeof COLUMN_TYPE_OPTIONS)[number];+const OptionDropdown = (props: OptionDropdownProps) => {const {isOpened,@@ -182,12 +196,30 @@ const OptionDropdown = (props: OptionDropdownProps) => {hideTableColumn,copyTableColumn,setDisplayFormat,+ changeTableColumnType,+ typeChangeDataContainer,+ columnIndex = -1,deleteTableColumn,createFilterFromColumn,onStartRename} = props;const [showFormatter, setShowFormatter] = useState(false);+ const [showTypeSelector, setShowTypeSelector] = useState(false);const [confirmDelete, setConfirmDelete] = useState(false);+ const currentType = colMeta[column]?.type;+ const columnTypeOptions = useMemo<ColumnTypeOption[]>(() => {+ if (!COLUMN_TYPE_OPTIONS.some(option => option.type === currentType)) {+ return [];+ }+ if (!showTypeSelector || !changeTableColumnType || !typeChangeDataContainer) {+ return COLUMN_TYPE_OPTIONS;+ }+ const compatibleTypes = new Set(+ getCompatibleColumnTypes(typeChangeDataContainer, columnIndex, currentType)+ );+ return COLUMN_TYPE_OPTIONS.filter(option => compatibleTypes.has(option.type));+ }, [changeTableColumnType, columnIndex, currentType, showTypeSelector, typeChangeDataContainer]);+ const hasCompatibleAlternative = columnTypeOptions.some(option => option.type !== currentType);const onOptionSelected: (v: TableOption) => void = useCallback(({value}) => {switch (value) {@@ -218,6 +250,9 @@ const OptionDropdown = (props: OptionDropdownProps) => {case TABLE_OPTION.RENAME:onStartRename?.();break;+ case TABLE_OPTION.CHANGE_TYPE:+ setShowTypeSelector(true);+ return;case TABLE_OPTION.CREATE_FILTER:createFilterFromColumn?.(column);break;@@ -260,7 +295,8 @@ const OptionDropdown = (props: OptionDropdownProps) => {// cant use conditions because it creates a circular dependency// TODO: move condition clause out of default-settings TABLE_OPTION_LISTconst validToFormat = op.value !== TABLE_OPTION.FORMAT_COLUMN || formatLabels.length;- return (!op.condition || op.condition(props)) && validToFormat;+ const validToChangeType = op.value !== TABLE_OPTION.CHANGE_TYPE || hasCompatibleAlternative;+ return (!op.condition || op.condition(props)) && validToFormat && validToChangeType;}).map(op => ({...op,icon: TABLE_OPTION_LIST_ICONS[op.icon]@@ -268,6 +304,7 @@ const OptionDropdown = (props: OptionDropdownProps) => {const onClose = useCallback(() => {setShowFormatter(false);+ setShowTypeSelector(false);setConfirmDelete(false);toggleMoreOptions(column);}, [column, toggleMoreOptions]);@@ -312,6 +349,26 @@ const OptionDropdown = (props: OptionDropdownProps) => {setDisplayFormat={setDisplayFormat}onClose={onClose}/>+ <Portaled+ left={120}+ top={-10}+ isOpened={Boolean(isOpened && showTypeSelector)}+ onClose={onClose}+ >+ <StyledPopover className="column-type-popover">+ <DropdownList+ options={columnTypeOptions}+ selectionIndex={columnTypeOptions.findIndex(option => option.type === currentType)}+ displayOption={option => (option as ColumnTypeOption).display}+ onOptionSelected={option => {+ const {type} = option as ColumnTypeOption;+ if (type !== currentType) changeTableColumnType?.(column, type);+ onClose();+ }}+ light+ />+ </StyledPopover>+ </Portaled></StyledOptionsDropdown></Portaled>);diff --git a/src/components/src/modal-container.tsx b/src/components/src/modal-container.tsxindex eabbe7537..71c8dda95 100644--- a/src/components/src/modal-container.tsx+++ b/src/components/src/modal-container.tsx@@ -456,6 +456,7 @@ export default function ModalContainerFactory(showAllTableColumns={visStateActions.showAllTableColumns}copyTableColumn={visStateActions.copyTableColumn}renameTableColumn={visStateActions.renameTableColumn}+ changeTableColumnType={visStateActions.changeTableColumnType}deleteTableColumn={visStateActions.deleteTableColumn}createFilterFromColumn={(id, column) =>visStateActions.createOrUpdateFilter(undefined, id, column)diff --git a/src/components/src/modals/data-table-modal-column-edit.spec.tsx b/src/components/src/modals/data-table-modal-column-edit.spec.tsxindex 4e21bd00b..7bd60ebde 100644--- a/src/components/src/modals/data-table-modal-column-edit.spec.tsx+++ b/src/components/src/modals/data-table-modal-column-edit.spec.tsx@@ -19,13 +19,22 @@ const tableProps: {current: Record<string, unknown> | null} = {current: null};function StubDataTable(props: {createFilterFromColumn?: (column: string) => void;renameTableColumn?: (column: string, newName: string) => void;+ changeTableColumnType?: (column: string, newType: string) => void;deleteTableColumn?: (column: string) => void;}) {tableProps.current = props;return (- <button type="button" onClick={() => props.createFilterFromColumn?.('region')}>- create-filter- </button>+ <>+ <button type="button" onClick={() => props.createFilterFromColumn?.('region')}>+ create-filter+ </button>+ <button+ type="button"+ onClick={() => props.changeTableColumnType?.('region', ALL_FIELD_TYPES.integer)}+ >+ change-type+ </button>+ </>);}@@ -51,6 +60,7 @@ function makeDataset(type: string, hiddenColumns: string[] = []) {function renderModal(type: string, hiddenColumns: string[] = []) {const createFilterFromColumn = jest.fn();const renameTableColumn = jest.fn();+ const changeTableColumnType = jest.fn();const deleteTableColumn = jest.fn();const toggleSidePanel = jest.fn();const toggleModal = jest.fn();@@ -63,6 +73,7 @@ function renderModal(type: string, hiddenColumns: string[] = []) {pinTableColumn={jest.fn()}copyTableColumn={jest.fn()}renameTableColumn={renameTableColumn}+ changeTableColumnType={changeTableColumnType}deleteTableColumn={deleteTableColumn}createFilterFromColumn={createFilterFromColumn}showDatasetTable={jest.fn()}@@ -71,7 +82,13 @@ function renderModal(type: string, hiddenColumns: string[] = []) {uiState={{} as never}/>);- return {...view, createFilterFromColumn, toggleSidePanel, toggleModal};+ return {+ ...view,+ createFilterFromColumn,+ changeTableColumnType,+ toggleSidePanel,+ toggleModal+ };}describe('data table column edits', () => {@@ -83,22 +100,28 @@ describe('data table column edits', () => {});test('create filter opens the filter panel and closes the table', () => {- const {createFilterFromColumn, toggleSidePanel, toggleModal} = renderModal('local');+ const {createFilterFromColumn, changeTableColumnType, toggleSidePanel, toggleModal} =+ renderModal('local');expect(tableProps.current?.renameTableColumn).toEqual(expect.any(Function));+ expect(tableProps.current?.changeTableColumnType).toEqual(expect.any(Function));expect(tableProps.current?.deleteTableColumn).toEqual(expect.any(Function));fireEvent.click(screen.getByRole('button', {name: 'create-filter'}));expect(createFilterFromColumn).toHaveBeenCalledWith('cities', 'region');expect(toggleSidePanel).toHaveBeenCalledWith('filter');expect(toggleModal).toHaveBeenCalledWith(null);++ fireEvent.click(screen.getByRole('button', {name: 'change-type'}));+ expect(changeTableColumnType).toHaveBeenCalledWith('cities', 'region', 'integer');});- test('externally hosted datasets can create a filter but not rename or delete', () => {+ test('externally hosted datasets cannot rename, re-type, or delete columns', () => {renderModal('externally-hosted');expect(tableProps.current?.createFilterFromColumn).toEqual(expect.any(Function));expect(tableProps.current?.renameTableColumn).toBeUndefined();+ expect(tableProps.current?.changeTableColumnType).toBeUndefined();expect(tableProps.current?.deleteTableColumn).toBeUndefined();});diff --git a/src/components/src/modals/data-table-modal.tsx b/src/components/src/modals/data-table-modal.tsxindex cc8fd75db..4810eff34 100644--- a/src/components/src/modals/data-table-modal.tsx+++ b/src/components/src/modals/data-table-modal.tsx@@ -254,6 +254,7 @@ interface DataTableModalProps {showAllTableColumns?: (id: string) => void;copyTableColumn: (id: string, column: string) => void;renameTableColumn?: (id: string, column: string, newName: string) => void;+ changeTableColumnType?: (id: string, column: string, newType: string) => void;deleteTableColumn?: (id: string, column: string) => void;createFilterFromColumn?: (id: string, column: string) => void;datasets: Datasets;@@ -285,6 +286,7 @@ function DataTableModalFactory(showAllTableColumns,copyTableColumn: copyTableColumnProp,renameTableColumn,+ changeTableColumnType,deleteTableColumn,createFilterFromColumn,datasets,@@ -435,6 +437,13 @@ function DataTableModalFactory([deleteTableColumn, dataId]);+ const handleChangeTableColumnType = useCallback(+ (column: string, newType: string) => {+ changeTableColumnType?.(dataId, column, newType);+ },+ [changeTableColumnType, dataId]+ );+const handleCreateFilterFromColumn = useCallback((column: string) => {createFilterFromColumn?.(dataId, column);@@ -722,6 +731,7 @@ function DataTableModalFactory(colMeta={colMeta}cellSizeCache={cellSizeCache}dataContainer={displayData.dataContainer}+ typeChangeDataContainer={activeDataset.dataContainer}pinnedColumns={(activeDataset.pinnedColumns || []).filter(column => !hiddenColumns.includes(column))}@@ -735,6 +745,9 @@ function DataTableModalFactory(renameTableColumn={canMutateColumns && renameTableColumn ? handleRenameTableColumn : undefined}+ changeTableColumnType={+ canMutateColumns && changeTableColumnType ? handleChangeTableColumnType : undefined+ }deleteTableColumn={canMutateColumns && deleteTableColumn ? handleDeleteTableColumn : undefined}diff --git a/src/constants/src/default-settings.ts b/src/constants/src/default-settings.tsindex 217604142..554c819d9 100644--- a/src/constants/src/default-settings.ts+++ b/src/constants/src/default-settings.ts@@ -541,6 +541,7 @@ export const TABLE_OPTION = keyMirror({FORMAT_COLUMN: null,HIDE: null,RENAME: null,+ CHANGE_TYPE: null,DELETE: null,CREATE_FILTER: null});@@ -556,6 +557,7 @@ export type TableOption = {| 'FORMAT_COLUMN'| 'HIDE'| 'RENAME'+ | 'CHANGE_TYPE'| 'DELETE'| 'CREATE_FILTER';display: string;@@ -612,6 +614,12 @@ export const TABLE_OPTION_LIST: TableOption[] = [icon: 'Edit',condition: props => props.renameTableColumn},+ {+ value: TABLE_OPTION.CHANGE_TYPE,+ display: 'Change Data Type',+ icon: 'Edit',+ condition: props => props.changeTableColumnType+ },{value: TABLE_OPTION.CREATE_FILTER,display: 'Create Filter',diff --git a/src/reducers/src/column-edit-reconcile.ts b/src/reducers/src/column-edit-reconcile.tsindex caf32d475..e6233f9dc 100644--- a/src/reducers/src/column-edit-reconcile.ts+++ b/src/reducers/src/column-edit-reconcile.ts@@ -2,16 +2,29 @@// Copyright contributors to the kepler.gl projectimport {ChartConfig, LayerChartConfig} from '@kepler.gl/types';+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';type NamedAxis = {field?: {name: string; type?: string} | null;title?: string | null;+ aggregation?: string | null;+ interval?: string | null;};+const BIN_AGGREGATIONS = new Set(['uniqueBin', 'numericBin', 'timeBin']);+const TIME_FIELD_TYPES = new Set<string>([ALL_FIELD_TYPES.timestamp, ALL_FIELD_TYPES.date]);++function binAggregationForType(type: string): string {+ if (TIME_FIELD_TYPES.has(type)) {+ return 'timeBin';+ }+ return type === ALL_FIELD_TYPES.real ? 'numericBin' : 'uniqueBin';+}+function followFieldName(name: string | null | undefined,renames: Record<string, string>,- alive: Set<string>+ alive: ReadonlyMap<string, {type: string}>): string | null {if (!name) {return null;@@ -23,7 +36,7 @@ function followFieldName(function followAxis<T extends NamedAxis>(axis: T | undefined,renames: Record<string, string>,- alive: Set<string>+ alive: ReadonlyMap<string, {type: string}>): T | undefined {if (!axis?.field?.name) {return axis;@@ -32,13 +45,22 @@ function followAxis<T extends NamedAxis>(if (!name) {return {...axis, field: null, title: null};}- if (name === axis.field.name) {+ const field = alive.get(name);+ const typeChanged = field?.type !== axis.field.type;+ if (name === axis.field.name && !typeChanged) {return axis;}+ const aggregation =+ typeChanged && axis.aggregation && BIN_AGGREGATIONS.has(axis.aggregation)+ ? binAggregationForType(field?.type || '')+ : axis.aggregation;+ const clearInterval = typeChanged && aggregation !== 'timeBin' && axis.interval !== undefined;return {...axis,- field: {...axis.field, name},- title: axis.title === axis.field.name ? name : axis.title+ field: {...axis.field, name, type: field?.type},+ title: axis.title === axis.field.name ? name : axis.title,+ aggregation,+ ...(clearInterval ? {interval: undefined} : {})};}@@ -50,7 +72,7 @@ export function remapChartsForDataset(charts: ChartConfig[] | undefined,dataId: string,renames: Record<string, string>,- alive: Set<string>+ alive: ReadonlyMap<string, {type: string}>): ChartConfig[] {return (charts || []).map(chart => {if (chart.dataId !== dataId) {diff --git a/src/reducers/src/column-edit-updaters.spec.ts b/src/reducers/src/column-edit-updaters.spec.tsindex 67c6e131c..01292894e 100644--- a/src/reducers/src/column-edit-updaters.spec.ts+++ b/src/reducers/src/column-edit-updaters.spec.ts@@ -1,16 +1,54 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import {ALL_FIELD_TYPES} from '@kepler.gl/constants';+import {+ ALL_FIELD_TYPES,+ ANIMATION_WINDOW,+ FILTER_TYPES,+ PLOT_TYPES,+ SORT_ORDER+} from '@kepler.gl/constants';import {VisStateActions} from '@kepler.gl/actions';import {KeplerTable} from '@kepler.gl/table';import SchemaManager, {VisState} from '@kepler.gl/schemas';+import {drainTasksForTesting, errorTaskInTest, succeedTaskInTest} from '@kepler.gl/tasks';+import type {PolygonFilter, RangeFilter, TimeRangeFilter} from '@kepler.gl/types';import visStateReducer from './vis-state';import {INITIAL_VIS_STATE} from './vis-state-updaters';const reduceVisState = visStateReducer as (state: VisState, action: unknown) => VisState;+function makeTable(id: string, data: Parameters<KeplerTable['updateSchema']>[0]) {+ const table = new KeplerTable({+ info: {id, label: id, type: 'local'},+ color: [0, 0, 0]+ });+ return table.updateSchema(data);+}++function withTables(...tables: KeplerTable[]): VisState {+ return {+ ...INITIAL_VIS_STATE,+ datasets: Object.fromEntries(tables.map(table => [table.id, table]))+ };+}++function withFilter(state: VisState, dataId: string, column: string): VisState {+ return reduceVisState(state, VisStateActions.createOrUpdateFilter(undefined, dataId, column));+}++function withSyncedFilter(primary: KeplerTable, secondary: KeplerTable, column: string): VisState {+ return reduceVisState(+ withFilter(withTables(primary, secondary), primary.id, column),+ VisStateActions.setFilter(0, ['dataId', 'name'], [secondary.id, column], 1)+ );+}++function changeType(state: VisState, dataId: string, column: string, type: string): VisState {+ return reduceVisState(state, VisStateActions.changeTableColumnType(dataId, column, type));+}+function makeCities() {const table = new KeplerTable({info: {id: 'cities', label: 'cities', type: 'local'},@@ -93,6 +131,10 @@ function withCities(): VisState {}describe('column edits', () => {+ beforeEach(() => {+ drainTasksForTesting();+ });+test('rename changes the field identity stored in layers, filters, and saved maps', () => {let state = withCities();state.datasets.cities.hiddenColumns = ['region', 'pop'];@@ -171,6 +213,474 @@ describe('column edits', () => {expect(kept.datasets.only.fields.map(field => field.name)).toEqual(['region']);});+ test('deleting an unrelated column preserves an inverted numeric histogram', () => {+ const table = makeTable('data', {+ fields: [+ {name: 'unused', type: ALL_FIELD_TYPES.string},+ {name: 'value', type: ALL_FIELD_TYPES.integer},+ {name: 'category', type: ALL_FIELD_TYPES.string}+ ],+ rows: [+ ['x', 1, 'A'],+ ['x', 2, 'B'],+ ['x', 3, 'A']+ ]+ });+ let state = withFilter(withTables(table), 'data', 'category');+ state = reduceVisState(state, VisStateActions.setFilter(0, 'value', ['A']));+ state = withFilter(state, 'data', 'value');+ state = reduceVisState(state, VisStateActions.setFilter(1, 'value', [2, 2]));+ state = reduceVisState(state, VisStateActions.setFilter(1, 'inverted', true));+ const histogramCount = () =>+ (state.filters[1] as RangeFilter).bins?.data?.reduce((sum, bin) => sum + bin.count, 0);+ expect(histogramCount()).toBe(2);++ state = reduceVisState(state, VisStateActions.deleteTableColumn('data', 'unused'));++ expect(histogramCount()).toBe(2);+ expect(state.datasets.data.filteredIndex).toEqual([0, 2]);+ expect(state.datasets.data.fields.map(field => field.type)).toEqual([+ ALL_FIELD_TYPES.integer,+ ALL_FIELD_TYPES.string+ ]);+ });++ test('type change converts values and reconciles incompatible references', () => {+ let state = withCities();+ state = reduceVisState(+ state,+ VisStateActions.addLayer({+ id: 'cities-heatmap',+ type: 'heatmap',+ config: {+ dataId: 'cities',+ label: 'cities heatmap',+ color: [255, 0, 0],+ columns: {lat: 'lat', lng: 'lng'},+ weightField: {name: 'pop', type: 'integer'},+ textLabel: [{field: {name: 'pop', type: 'integer'}}]+ }+ })+ );+ state = changeType(state, 'cities', 'pop', ALL_FIELD_TYPES.string);++ const pop = state.datasets.cities.fields.find(field => field.name === 'pop');+ expect(pop).toMatchObject({type: ALL_FIELD_TYPES.string, analyzerType: 'STRING', format: ''});+ expect(state.datasets.cities.dataContainer.rowAsArray(0)[3]).toBe('10');+ expect(state.layers).toHaveLength(2);+ expect(+ (state.layers.find(layer => layer.id === 'cities-heatmap')?.config as {weightField?: unknown})+ .weightField+ ).toBeNull();+ expect(+ state.layers.find(layer => layer.id === 'cities-heatmap')?.config.textLabel[0].field+ ).toMatchObject({name: 'pop', type: 'string'});+ expect(state.charts[0].yAxis?.field).toEqual({name: 'pop', type: 'string'});+ });++ test('type change preserves polygon filters targeting the dataset', () => {+ let state = withCities();+ const polygonFilter: PolygonFilter = {+ id: 'polygon-filter',+ dataId: ['cities'],+ enabled: true,+ fixedDomain: true,+ view: 'side',+ isAnimating: false,+ speed: 1,+ name: [],+ type: FILTER_TYPES.polygon,+ fieldIdx: [],+ domain: null,+ value: {+ type: 'Feature',+ id: 'polygon-selection',+ properties: {filterId: 'polygon-filter'},+ geometry: {+ type: 'Polygon',+ coordinates: [+ [+ [-123, 37],+ [-122, 37],+ [-122, 38],+ [-123, 37]+ ]+ ]+ }+ },+ plotType: {type: PLOT_TYPES.histogram},+ yAxis: null,+ layerId: ['cities-layer'],+ gpu: false+ };++ state = {...state, filters: [...state.filters, polygonFilter]};+ state = changeType(state, 'cities', 'pop', ALL_FIELD_TYPES.string);++ expect(state.filters.find(filter => filter.id === polygonFilter.id)).toBe(polygonFilter);++ state = reduceVisState(state, VisStateActions.deleteTableColumn('cities', 'lat'));++ expect(state.layers).toHaveLength(0);+ expect(state.filters.find(filter => filter.id === polygonFilter.id)).toBeUndefined();+ expect(state.datasets.cities.filterRecord?.gpu).not.toContainEqual(+ expect.objectContaining({id: polygonFilter.id})+ );+ expect(state.datasets.cities.filterRecord?.cpu).not.toContainEqual(+ expect.objectContaining({id: polygonFilter.id})+ );+ });++ test('type change preserves an existing sort on another column', () => {+ let state = withCities();+ state = reduceVisState(+ state,+ VisStateActions.sortTableColumn('cities', 'region', SORT_ORDER.ASCENDING)+ );+ expect(state.datasets.cities.sortColumn).toEqual({region: SORT_ORDER.ASCENDING});+ expect(state.datasets.cities.sortOrder).toEqual([1, 0]);++ state = changeType(state, 'cities', 'pop', ALL_FIELD_TYPES.string);++ expect(state.datasets.cities.sortColumn).toEqual({region: SORT_ORDER.ASCENDING});+ expect(state.datasets.cities.sortOrder).toEqual([1, 0]);+ });++ test('type change clears timeline state when a time filter becomes non-time', () => {+ const events = makeTable('events', {+ fields: [+ {name: 'start', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'DATETIME'},+ {name: 'amount', type: ALL_FIELD_TYPES.integer}+ ],+ rows: [+ ['2024-01-01T00:00:00Z', 10],+ ['2024-01-02T00:00:00Z', 20]+ ]+ });+ let state = withFilter(withTables(events), 'events', 'start');+ state = {+ ...state,+ filters: [+ {+ ...(state.filters[0] as TimeRangeFilter),+ syncedWithLayerTimeline: true,+ syncTimelineMode: 1 as const,+ isAnimating: true,+ animationWindow: ANIMATION_WINDOW.interval,+ timeBins: {events: {day: []}},+ lineChart: {+ series: [],+ yDomain: [0, 1],+ xDomain: [0, 1],+ aggregation: 'sum',+ interval: 'day',+ yAxis: 'amount'+ },+ timezone: 'UTC',+ invertTrendColor: true,+ showTimeDisplay: true,+ yAxis: state.datasets.events.fields[1],+ plotType: {type: PLOT_TYPES.lineChart, interval: 'day', aggregation: 'sum'}+ }+ ]+ };++ state = changeType(state, 'events', 'start', ALL_FIELD_TYPES.string);++ const filter = state.filters[0];+ expect(filter.type).toBe('multiSelect');+ expect(filter).not.toHaveProperty('syncedWithLayerTimeline');+ expect(filter).not.toHaveProperty('syncTimelineMode');+ expect(filter).not.toHaveProperty('timeBins');+ expect(filter).not.toHaveProperty('lineChart');+ expect(filter).not.toHaveProperty('timezone');+ expect(filter).not.toHaveProperty('invertTrendColor');+ expect(filter).not.toHaveProperty('showTimeDisplay');+ expect(filter).toMatchObject({+ fixedDomain: false,+ isAnimating: false,+ animationWindow: ANIMATION_WINDOW.free,+ yAxis: null,+ plotType: {type: PLOT_TYPES.histogram}+ });+ });++ test('type change detaches an incompatible synced filter member and refilters its dataset', () => {+ const primary = makeTable('primary', {+ fields: [{name: 'value', type: ALL_FIELD_TYPES.integer}],+ rows: [[1], [2]]+ });+ const secondary = makeTable('secondary', {+ fields: [{name: 'value', type: ALL_FIELD_TYPES.integer}],+ rows: [[1], [3]]+ });++ let state = withSyncedFilter(primary, secondary, 'value');+ expect(state.filters[0].dataId).toEqual(['primary', 'secondary']);+ expect(state.datasets.secondary.filterRecord?.gpu).toHaveLength(1);++ state = changeType(state, 'secondary', 'value', ALL_FIELD_TYPES.string);++ expect(state.filters[0].dataId).toEqual(['primary']);+ expect(state.filters[0].name).toEqual(['value']);+ expect(state.filters[0].type).toBe('range');+ expect(state.datasets.secondary.filterRecord).toMatchObject({+ dynamicDomain: [],+ fixedDomain: [],+ cpu: [],+ gpu: []+ });+ });++ test('changing a synced secondary dataset does not reconcile the primary plot axis', () => {+ const makeEvents = (id: string) =>+ makeTable(id, {+ fields: [+ {name: 'start', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'TIME', format: 'x'},+ {name: 'amount', type: ALL_FIELD_TYPES.integer}+ ],+ rows: [+ [1_700_000_000, 10],+ [1_700_000_020, 20]+ ]+ });+ const primary = makeEvents('primary');+ const secondary = makeEvents('secondary');+ let state = withSyncedFilter(primary, secondary, 'start');+ state = reduceVisState(+ state,+ VisStateActions.setFilterPlot(0, {yAxis: state.datasets.primary.fields[1]})+ );++ state = changeType(state, 'secondary', 'amount', ALL_FIELD_TYPES.string);++ expect(state.filters[0].dataId).toEqual(['primary', 'secondary']);+ expect(state.filters[0].yAxis).toMatchObject({name: 'amount', type: 'integer'});+ expect(state.filters[0].plotType.type).toBe('lineChart');+ });++ test('detaching a synced primary dataset rebuilds the plot for the new primary', () => {+ const primary = makeTable('primary', {+ fields: [+ {name: 'start', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'TIME', format: 'x'},+ {name: 'amount', type: ALL_FIELD_TYPES.integer}+ ],+ rows: [+ [1_700_000_000, 10],+ [1_700_000_020, 20]+ ]+ });+ const secondary = makeTable('secondary', {+ fields: [{name: 'start', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'TIME', format: 'x'}],+ rows: [[1_700_000_040], [1_700_000_060]]+ });+ let state = withSyncedFilter(primary, secondary, 'start');+ state = reduceVisState(+ state,+ VisStateActions.setFilterPlot(0, {yAxis: state.datasets.primary.fields[1]})+ );+ expect(state.filters[0].lineChart).toBeDefined();++ state = changeType(state, 'primary', 'start', ALL_FIELD_TYPES.string);++ expect(state.filters[0].dataId).toEqual(['secondary']);+ expect(state.filters[0].name).toEqual(['start']);+ expect(state.filters[0].type).toBe('timeRange');+ expect(state.filters[0].yAxis).toBeNull();+ expect(state.filters[0].plotType.type).toBe('histogram');+ expect(state.filters[0].lineChart).toBeUndefined();+ });++ test('grouped timeline series survive a rename and rebuild when group values collapse', () => {+ const events = makeTable('events', {+ fields: [+ {name: 'start', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'TIME', format: 'x'},+ {name: 'group', type: ALL_FIELD_TYPES.string},+ {name: 'amount', type: ALL_FIELD_TYPES.integer}+ ],+ rows: [+ [1_700_000_000, '01', 10],+ [1_700_000_020, '1', 20]+ ]+ });+ let state = withFilter(withTables(events), 'events', 'start');+ state = reduceVisState(+ state,+ VisStateActions.setFilterPlot(0, {yAxis: state.datasets.events.fields[2]})+ );+ state = reduceVisState(+ state,+ VisStateActions.setFilterPlot(0, {+ plotType: {groupBy: {fieldName: 'group', numGroups: 'ALL', groupOthers: false}}+ })+ );+ const seriesBefore = state.filters[0].lineChart?.series as {names?: string[]};+ expect(new Set(seriesBefore.names)).toEqual(new Set(['01', '1']));++ state = reduceVisState(state, VisStateActions.renameTableColumn('events', 'group', 'category'));+ expect(state.filters[0].lineChart?.series).toEqual(seriesBefore);+ expect(state.filters[0].plotType.groupBy.fieldName).toBe('category');+ expect(state.datasets.events.fields[1].type).toBe(ALL_FIELD_TYPES.string);++ state = changeType(state, 'events', 'category', ALL_FIELD_TYPES.integer);++ const namesAfter = (state.filters[0].lineChart?.series as {names?: string[]}).names;+ expect(new Set(namesAfter)).toEqual(new Set(['1']));+ });++ test('type change clears an interval end field that is no longer a timestamp', () => {+ const events = makeTable('events', {+ fields: [+ {name: 'start', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'TIME', format: 'x'},+ {name: 'end', type: ALL_FIELD_TYPES.timestamp, analyzerType: 'TIME', format: 'x'},+ {name: 'value', type: ALL_FIELD_TYPES.integer}+ ],+ rows: [+ [1_700_000_000, 1_700_086_400, 10],+ [1_700_000_020, 1_700_172_800, 20]+ ]+ });+ let state = withTables(events);+ state = reduceVisState(+ state,+ VisStateActions.addChart({+ id: 'events-chart',+ title: 'Events',+ type: 'barChart',+ dataId: 'events',+ applyFilters: true,+ display: {},+ chartDisplay: {},+ xAxis: {+ field: {name: 'end', type: 'timestamp'},+ aggregation: 'timeBin',+ interval: 'day',+ title: 'end'+ },+ yAxis: {field: {name: 'value', type: 'integer'}, aggregation: 'sum', title: 'value'}+ })+ );+ state = withFilter(state, 'events', 'start');+ state = reduceVisState(state, VisStateActions.setFilter(0, 'endName', 'end', 0));+ state = reduceVisState(+ state,+ VisStateActions.setFilterPlot(0, {yAxis: state.datasets.events.fields[2]})+ );+ expect(state.filters[0].endName).toEqual(['end']);+ expect(state.filters[0].yAxis?.name).toBe('value');++ state = changeType(state, 'events', 'value', ALL_FIELD_TYPES.string);+ expect(state.filters[0].yAxis).toBeNull();+ expect(state.filters[0].plotType.type).toBe('histogram');+ expect(state.filters[0].endName).toEqual(['end']);+ expect(state.charts[0].yAxis).toMatchObject({+ field: {name: 'value', type: 'string'},+ aggregation: 'sum'+ });+ let timeFilter = state.filters[0] as TimeRangeFilter;+ const interval = timeFilter.plotType.interval;+ const intervalBins = timeFilter.timeBins?.events?.[interval];+ expect(intervalBins?.reduce((sum, bin) => sum + bin.count, 0)).toBeGreaterThan(2);++ state = changeType(state, 'events', 'end', ALL_FIELD_TYPES.string);+ expect(state.filters[0].type).toBe('timeRange');+ expect(state.filters[0].endName).toBeUndefined();+ expect(state.filters[0].endFieldIdx).toBeUndefined();+ timeFilter = state.filters[0] as TimeRangeFilter;+ const bins = timeFilter.timeBins?.events?.[interval];+ expect(bins?.reduce((sum, bin) => sum + bin.count, 0)).toBe(2);+ expect(state.charts[0].xAxis).toMatchObject({+ field: {name: 'end', type: 'string'},+ aggregation: 'uniqueBin'+ });+ expect(state.charts[0].xAxis?.interval).toBeUndefined();+ });++ test('invalid type change is atomic and schedules a precise error', () => {+ const state = withCities();+ const next = changeType(state, 'cities', 'region', ALL_FIELD_TYPES.integer);++ expect(next.datasets.cities).toBe(state.datasets.cities);+ expect(next.datasets.cities.fields.find(field => field.name === 'region')?.type).toBe(+ ALL_FIELD_TYPES.string+ );+ expect(next.datasets.cities.dataContainer.rowAsArray(0)[2]).toBe('west');++ const tasks = drainTasksForTesting();+ expect(tasks).toHaveLength(1);+ const notification = succeedTaskInTest(tasks[0], undefined);+ expect(notification.payload.message).toContain(+ 'Cannot change “region” to integer: row 1 value “west” is not a safe integer'+ );+ expect(notification.payload.message).toContain('No values were changed.');+ });++ test('late column statistics cannot overwrite a changed column', () => {+ let state = withCities();+ state = reduceVisState(state, VisStateActions.loadColumnStats('cities', 'pop'));+ const requestRevision = state.datasets.cities.dataRevision;+ const [task] = drainTasksForTesting();+ const result = {type: 'numeric', mean: 25};+ const staleSuccess = succeedTaskInTest(task, {+ result,+ filterProps: {type: FILTER_TYPES.range}+ });+ expect(staleSuccess).toMatchObject({+ fieldType: ALL_FIELD_TYPES.integer+ });++ state = changeType(state, 'cities', 'pop', ALL_FIELD_TYPES.string);+ expect(state.datasets.cities.dataRevision).toBe(requestRevision + 1);+ const changedState = state;++ state = reduceVisState(state, staleSuccess);++ expect(state).toBe(changedState);+ const changedField = state.datasets.cities.fields.find(field => field.name === 'pop');+ expect(changedField).toMatchObject({type: ALL_FIELD_TYPES.string});+ expect(changedField).not.toHaveProperty('isLoadingStats');+ expect(+ state.datasets.cities.fields.find(field => field.name === 'pop')?.filterProps?.columnStats+ ).not.toBe(result);+ });++ test.each(['refresh', 'append'])('pending statistics complete after a normal %s', update => {+ let state = withCities();+ const types = state.datasets.cities.fields.map(field => field.type);+ state = reduceVisState(state, VisStateActions.loadColumnStats('cities', 'pop'));+ const [task] = drainTasksForTesting();+ state = reduceVisState(state, VisStateActions.pinTableColumn('cities', 'region'));+ const row = [41, -75, 'north', 50];+ state = reduceVisState(+ state,+ update === 'refresh'+ ? VisStateActions.refreshDatasetSuccess('cities', {+ notModified: false,+ data: {fields: state.datasets.cities.fields, rows: [row]}+ })+ : VisStateActions.addToDataset('cities', [row])+ );++ const result = {type: 'numeric', mean: 25};+ const successState = reduceVisState(+ state,+ succeedTaskInTest(task, {result, filterProps: {type: FILTER_TYPES.range}})+ );+ const logError = jest.spyOn(console, 'error').mockImplementation(() => undefined);+ const errorState = reduceVisState(state, errorTaskInTest(task, new Error('stats failed')));+ logError.mockRestore();++ expect(state.datasets.cities.fields.map(field => field.type)).toEqual(types);+ expect(successState.datasets.cities.fields[3]).toMatchObject({+ isLoadingStats: false,+ filterProps: {columnStats: result}+ });+ expect(errorState.datasets.cities.fields[3].isLoadingStats).toBe(false);+ expect(state.datasets.cities.dataContainer.rowAsArray(update === 'refresh' ? 0 : 2)).toEqual(+ row+ );+ });+test('duplicate rename and non-tabular datasets are ignored', () => {const state = withCities();const duplicate = reduceVisState(@@ -197,6 +707,7 @@ describe('column edits', () => {);expect(unchanged).toBe(tiled);expect(reduceVisState(tiled, VisStateActions.deleteTableColumn('tiles', 'pop'))).toBe(tiled);+ expect(changeType(tiled, 'tiles', 'pop', ALL_FIELD_TYPES.string)).toBe(tiled);});test('externally hosted datasets ignore rename and delete', () => {@@ -217,6 +728,7 @@ describe('column edits', () => {reduceVisState(state, VisStateActions.renameTableColumn('remote', 'region', 'area'))).toBe(state);expect(reduceVisState(state, VisStateActions.deleteTableColumn('remote', 'pop'))).toBe(state);+ expect(changeType(state, 'remote', 'pop', ALL_FIELD_TYPES.string)).toBe(state);expect(state.datasets.remote.fields[0].displayName).toBe('Region');});});diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.tsindex 1914221f8..a140b816e 100644--- a/src/reducers/src/vis-state-updaters.ts+++ b/src/reducers/src/vis-state-updaters.ts@@ -103,6 +103,7 @@ import {import {generateHashId, toArray, arrayMove} from '@kepler.gl/common-utils';// Mergersimport {+ ALL_FIELD_TYPES,ANIMATION_WINDOW,BASE_SPEED,COMPARE_TYPES,@@ -201,6 +202,8 @@ import {Datasets,KeplerTable,FilterProps,+ buildChangeColumnTypeData,+ formatColumnTypeConversionError,assignGpuChannel,buildDeleteColumnData,buildRenameColumnData,@@ -4410,16 +4413,14 @@ function reconcileLayerWithNewSchema(}});- // follow renamed text-label fields+ // follow renamed or re-typed text-label fieldsif (layer.config.textLabel?.length) {- const textLabel = layer.config.textLabel.map(tl =>- tl.field && renames[tl.field.name]- ? {...tl, field: {...tl.field, name: renames[tl.field.name]}}- : tl- );- if (textLabel.some((tl, i) => tl !== layer.config.textLabel[i])) {- configPatch.textLabel = textLabel;- }+ configPatch.textLabel = layer.config.textLabel.map(tl => {+ const name = tl.field?.name;+ const field =+ name && newTable.fields.find(candidate => candidate.name === (renames[name] ?? name));+ return field && (field.name !== name || field.type !== tl.field?.type) ? {...tl, field} : tl;+ });}// shallow-copy the layer so we never mutate redux state; the copy keeps the@@ -4440,6 +4441,57 @@ function reconcileLayerWithNewSchema(return validated && validated.hasAllColumns() ? validated : null;}+const FILTER_PLOT_Y_AXIS_TYPES = new Set<string>([ALL_FIELD_TYPES.integer, ALL_FIELD_TYPES.real]);++function clearTimelineFilterState(filter: Filter): Filter {+ const {+ syncedWithLayerTimeline: _syncedWithLayerTimeline,+ syncTimelineMode: _syncTimelineMode,+ timeBins: _timeBins,+ lineChart: _lineChart,+ timezone: _timezone,+ invertTrendColor: _invertTrendColor,+ showTimeDisplay: _showTimeDisplay,+ ...rest+ } = filter as TimeRangeFilter;++ return {+ ...rest,+ fixedDomain: false,+ isAnimating: false,+ animationWindow: ANIMATION_WINDOW.free,+ yAxis: null,+ plotType: {type: PLOT_TYPES.histogram}+ } as Filter;+}++function reconcileTimelinePlotAfterPrimaryRemoval(+ filter: Filter,+ datasets: VisState['datasets']+): Filter {+ if (filter.type !== FILTER_TYPES.timeRange) {+ return filter;+ }++ const primaryDataset = datasets[filter.dataId[0]];+ const yAxis = filter.yAxis?.name+ ? primaryDataset?.fields.find(+ field => field.name === filter.yAxis?.name && FILTER_PLOT_Y_AXIS_TYPES.has(field.type)+ ) ?? null+ : null;+ const groupByFieldName = filter.plotType?.groupBy?.fieldName;+ const hasGroupByField = groupByFieldName+ ? primaryDataset?.fields.some(field => field.name === groupByFieldName)+ : true;+ const plotType = {+ ...filter.plotType,+ ...(filter.yAxis && !yAxis ? {type: PLOT_TYPES.histogram} : {}),+ ...(!hasGroupByField ? {groupBy: null} : {})+ };++ return {...filter, yAxis, plotType, lineChart: undefined};+}+/*** Update an existing dataset's schema (columns/fields) in place and reconcile* everything that referenced the old columns: filters, layers, tooltip fields,@@ -4470,7 +4522,9 @@ export function updateDatasetUpdater(// 1. rebuild the table schema in place (same id/label/color/metadata)const newTable = copyTableAndUpdate(existing, {});+ const [sortedColumn, sortMode] = Object.entries(existing.sortColumn || {})[0] || [];newTable.updateSchema(data);+ newTable.dataRevision = existing.dataRevision + 1;// The data container is rebuilt unsorted and unfiltered; reset the table's// view state that indexes into the old container.@@ -4478,15 +4532,20 @@ export function updateDatasetUpdater(newTable.filterRecordCPU = undefined;newTable.sortColumn = undefined;newTable.sortOrder = null;- const newFieldsByName = new Set(newTable.fields.map(f => f.name));+ const newFieldsByName = new Map(newTable.fields.map(field => [field.name, field]));newTable.pinnedColumns = (existing.pinnedColumns || []).map(name => renames[name] ?? name).filter(name => newFieldsByName.has(name));newTable.hiddenColumns = (existing.hiddenColumns || []).map(name => renames[name] ?? name).filter(name => newFieldsByName.has(name));+ const nextSortedColumn = sortedColumn ? renames[sortedColumn] ?? sortedColumn : null;+ if (nextSortedColumn && sortMode && newFieldsByName.has(nextSortedColumn)) {+ sortDatasetByColumn(newTable, nextSortedColumn, sortMode);+ }const newDatasets = {...state.datasets, [dataId]: newTable};+ const datasetIdsToRefilter = new Set<string>([dataId]);// 2. reconcile filterslet newFilters = state.filters@@ -4496,57 +4555,92 @@ export function updateDatasetUpdater(return filter;}+ // Polygon filters target layers rather than dataset fields, so a column+ // type change should not reconcile them against the field list.+ if (filter.type === FILTER_TYPES.polygon) {+ return filter;+ }+const oldName = filter.name[vi];const newName = renames[oldName] ?? oldName;const fieldIdx = newTable.getColumnFieldIdx(newName);+ const nextFilterType = fieldIdx === -1 ? null : newTable.getColumnFilterProps(newName)?.type;- if (fieldIdx === -1) {- // column is gone: drop a single-dataset filter, splice it out of a synced one- return filter.dataId.length === 1- ? null- : _removeFilterDataIdAtValueIndex(filter, vi, newDatasets);+ if (+ fieldIdx === -1 ||+ (filter.dataId.length > 1 && nextFilterType && nextFilterType !== filter.type)+ ) {+ // Drop a missing field or detach an incompatible synced member.+ if (filter.dataId.length === 1) {+ return null;+ }+ filter.dataId.forEach(id => datasetIdsToRefilter.add(id));+ const detachedFilter = _removeFilterDataIdAtValueIndex(filter, vi, newDatasets);+ return vi === 0+ ? reconcileTimelinePlotAfterPrimaryRemoval(detachedFilter, newDatasets)+ : detachedFilter;}// column present (possibly renamed): re-resolve name/fieldIdx/domain- const {filter: updated} = applyFilterFieldName(filter, newDatasets, dataId, newName, vi, {+ let {filter: updated} = applyFilterFieldName(filter, newDatasets, dataId, newName, vi, {mergeDomain: vi > 0});if (!updated) {return null;}+ const leavingTimeFilter =+ filter.type === FILTER_TYPES.timeRange && updated.type !== FILTER_TYPES.timeRange;++ const oldEndName = toArray((filter as TimeRangeFilter).endName)[vi];+ if (updated.type === FILTER_TYPES.timeRange && oldEndName) {+ const endName = renames[oldEndName] ?? oldEndName;+ updated =+ applyTimeFilterEndFieldName(updated, newDatasets, dataId, endName, vi).filter ??+ applyTimeFilterEndFieldName(updated, newDatasets, dataId, null, vi).filter ??+ updated;+ }- // brush filters carry a second y-axis field; follow a rename there too- const yAxis =- updated.yAxis?.name &&- (renames[updated.yAxis.name]- ? {...updated.yAxis, name: renames[updated.yAxis.name]}- : newFieldsByName.has(updated.yAxis.name)- ? updated.yAxis- : null);+ // Time plots only accept numeric y-axis fields. Re-resolve the complete+ // field so type and fieldIdx also follow a schema change.+ let yAxis = updated.yAxis;+ if (vi === 0 && updated.yAxis?.name) {+ const yAxisName = renames[updated.yAxis.name] ?? updated.yAxis.name;+ const yAxisField = newFieldsByName.get(yAxisName);+ yAxis = yAxisField && FILTER_PLOT_Y_AXIS_TYPES.has(yAxisField.type) ? yAxisField : null;+ }- return {+ let nextFilter = {...updated,yAxis,+ ...(updated.yAxis && !yAxis+ ? {plotType: {...updated.plotType, type: PLOT_TYPES.histogram}}+ : {}),// clamp the user's previous selection into the new domain instead of// resetting it to the full rangevalue: adjustValueToFilterDomain(filter.value, updated)};+ if (leavingTimeFilter) {+ nextFilter = clearTimelineFilterState(nextFilter);+ } else if (vi === 0 && nextFilter.type === FILTER_TYPES.timeRange) {+ const groupBy = nextFilter.plotType.groupBy;+ if (groupBy && renames[groupBy.fieldName]) {+ nextFilter.plotType = {+ ...nextFilter.plotType,+ groupBy: {...groupBy, fieldName: renames[groupBy.fieldName]}+ };+ }+ // Timeline bins can remain identical while y-axis or group-by values+ // change, so do not reuse the old series cache after a data rebuild.+ nextFilter = {...nextFilter, lineChart: undefined};+ }+ // Keep existing plots; time plots are rebuilt after filtering below.+ return filter.type === nextFilter.type || nextFilter.type === FILTER_TYPES.timeRange+ ? nextFilter+ : updateFilterPlot(newDatasets, nextFilter, dataId, state.layers);}).filter(Boolean) as Filter[];- // reassign GPU channels now that the filter list is settled- newFilters = newFilters.map(filter =>- filter.gpu ? assignGpuChannel(setFilterGpuMode(filter, newFilters), newFilters) : filter- );-- // 3. rebuild the table's filter state (gpuFilter / filterRecord / filteredIndex)- newTable.filterTable(- newFilters,- state.layers.filter(l => l.config.dataId === dataId),- {}- );-- // 4. reconcile layers+ // 3. reconcile layersconst layersToRemove: string[] = [];const newLayers = state.layers.map(layer => {if (layer.config.dataId !== dataId) {@@ -4560,6 +4654,38 @@ export function updateDatasetUpdater(return layer; // placeholder, dropped via removeLayerUpdater below});+ // Polygon filters reference layers, not fields. Preserve them for schema+ // edits that keep their target layers, but remove invalid targets before+ // rebuilding filter records when a required layer column was deleted.+ const removedLayerIds = new Set(layersToRemove);+ if (removedLayerIds.size) {+ newFilters = newFilters.flatMap(filter => {+ if (+ filter.type !== FILTER_TYPES.polygon ||+ !filter.layerId?.some(id => removedLayerIds.has(id))+ ) {+ return [filter];+ }+ filter.dataId.forEach(id => datasetIdsToRefilter.add(id));+ const layerId = filter.layerId.filter(id => !removedLayerIds.has(id));+ return layerId.length ? [{...filter, layerId}] : [];+ });+ }++ // Reassign GPU channels now that the filter list is settled.+ newFilters = newFilters.map(filter =>+ filter.gpu ? assignGpuChannel(setFilterGpuMode(filter, newFilters), newFilters) : filter+ );++ // 4. rebuild the table's filter state using only layers that will survive+ const survivingLayers = newLayers.filter(layer => !removedLayerIds.has(layer.id));+ const filteredDatasets = applyFiltersToDatasets(+ Array.from(datasetIdsToRefilter),+ newDatasets,+ newFilters,+ survivingLayers+ );+// 5. reconcile tooltip fieldsToShowlet interactionConfig = state.interactionConfig;const tooltip = interactionConfig?.tooltip;@@ -4583,7 +4709,7 @@ export function updateDatasetUpdater(let nextState = {...state,- datasets: newDatasets,+ datasets: filteredDatasets,filters: newFilters,layers: newLayers,interactionConfig,@@ -4593,6 +4719,10 @@ export function updateDatasetUpdater(// 6. drop layers that lost their required columnsnextState = layersToRemove.reduce((accu, id) => removeLayerUpdater(accu, {id}), nextState);+ // Filtering and polygon-layer removal can change the rows behind timeline+ // plots, so rebuild their cached bins/series from the reconciled state.+ nextState = updateFilterPlotsAfterFiltering(nextState, Array.from(datasetIdsToRefilter));+// 7. recompute layer domains + data for the updated datasetnextState = updateAllLayerDomainData(nextState, dataId);@@ -4623,6 +4753,45 @@ export function renameTableColumnUpdater(});}+/**+ * Convert a complete column before committing its new type. A failed value+ * leaves the table and every reference unchanged.+ */+export function changeTableColumnTypeUpdater(+ state: VisState,+ action: VisStateActions.ChangeTableColumnTypeUpdaterAction+): VisState {+ const {dataId, fieldName, newType} = action;+ const dataset = state.datasets[dataId];+ if (!dataset || !canMutateDatasetSchema(dataset)) {+ return state;+ }+ const result = buildChangeColumnTypeData(dataset, fieldName, newType);+ if (!result) {+ return state;+ }+ if (result.error) {+ return withTask(+ state,+ ACTION_TASK_ADD_NOTIFICATION().map(() =>+ addNotification(+ errorNotification({+ id: `column-type-${dataId}-${fieldName}`,+ message: `Cannot change “${fieldName}” to ${newType}: ${formatColumnTypeConversionError(+ result.error+ )}. No values were changed.`+ })+ )+ )+ );+ }+ return updateDatasetUpdater(state, {+ dataId,+ data: result.payload.data,+ renames: result.payload.renames+ });+}+/*** Drop a column and the layers, filters, and other references that required it.* The last remaining column is kept.@@ -5930,8 +6099,14 @@ export function loadColumnStatsUpdater(const tasks = fieldsToLoad.map(f =>LOAD_COLUMN_STATS_TASK({dataset: newDataset, field: f}).bimap(- ({result, filterProps}) => loadColumnStatsSuccess(newDataset.id, f.name, result, filterProps),- (error: Error) => loadColumnStatsError(newDataset.id, f.name, error)+ ({result, filterProps}) =>+ loadColumnStatsSuccess(newDataset.id, f.name, result, filterProps, {+ fieldType: f.type+ }),+ (error: Error) =>+ loadColumnStatsError(newDataset.id, f.name, error, {+ fieldType: f.type+ })));const nextState = pick_('datasets')(merge_({[dataId]: newDataset}))(state);@@ -5946,11 +6121,17 @@ export function loadColumnStatsUpdater(*/export function loadColumnStatsSuccessUpdater(state: VisState,- {result, filterProps, dataId, fieldName}: VisStateActions.LoadColumnStatsSuccessUpdaterAction+ {+ result,+ filterProps,+ dataId,+ fieldName,+ fieldType+ }: VisStateActions.LoadColumnStatsSuccessUpdaterAction): VisState {const dataset = state.datasets[dataId];const field = (dataset?.fields || []).find(f => f.name === fieldName);- if (!dataset || !field) {+ if (!dataset || !field || (fieldType !== undefined && field.type !== fieldType)) {return state;}@@ -5972,14 +6153,14 @@ export function loadColumnStatsSuccessUpdater(*/export function loadColumnStatsErrorUpdater(state: VisState,- {error, dataId, fieldName}: VisStateActions.LoadColumnStatsErrorUpdaterAction+ {error, dataId, fieldName, fieldType}: VisStateActions.LoadColumnStatsErrorUpdaterAction): VisState {- Console.error(error);const dataset = state.datasets[dataId];const field = (dataset?.fields || []).find(f => f.name === fieldName);- if (!dataset || !field) {+ if (!dataset || !field || (fieldType !== undefined && field.type !== fieldType)) {return state;}+ Console.error(error);const newFields = dataset.fields.map(f =>f.name === fieldName ? {...f, isLoadingStats: false} : f);diff --git a/src/reducers/src/vis-state.ts b/src/reducers/src/vis-state.tsindex 027e18fbf..f016ab5c1 100644--- a/src/reducers/src/vis-state.ts+++ b/src/reducers/src/vis-state.ts@@ -147,6 +147,8 @@ const actionHandler = {[ActionTypes.RENAME_TABLE_COLUMN]: visStateUpdaters.renameTableColumnUpdater,+ [ActionTypes.CHANGE_TABLE_COLUMN_TYPE]: visStateUpdaters.changeTableColumnTypeUpdater,+[ActionTypes.DELETE_TABLE_COLUMN]: visStateUpdaters.deleteTableColumnUpdater,[ActionTypes.SET_FEATURES]: visStateUpdaters.setFeaturesUpdater,diff --git a/src/table/src/dataset-ops/column-edit.spec.ts b/src/table/src/dataset-ops/column-edit.spec.tsindex 9c8dfdaf0..6e15ed6b5 100644--- a/src/table/src/dataset-ops/column-edit.spec.ts+++ b/src/table/src/dataset-ops/column-edit.spec.ts@@ -2,8 +2,15 @@// Copyright contributors to the kepler.gl projectimport {ALL_FIELD_TYPES} from '@kepler.gl/constants';+import {createArrowVector} from '@kepler.gl/utils';import KeplerTable from '../kepler-table';-import {buildDeleteColumnData, buildRenameColumnData} from './column-edit';+import {+ buildChangeColumnTypeData,+ buildDeleteColumnData,+ buildRenameColumnData,+ formatColumnTypeConversionError,+ getCompatibleColumnTypes+} from './column-edit';function makeTable() {const table = new KeplerTable({@@ -23,6 +30,22 @@ function makeTable() {return table;}+function makeColumnTable(+ values: unknown[],+ type: string = ALL_FIELD_TYPES.string,+ analyzerType?: string+) {+ const table = new KeplerTable({+ info: {id: 'typed', label: 'typed', type: 'local'},+ color: [0, 0, 0]+ });+ table.updateSchema({+ fields: [{name: 'value', type, analyzerType}],+ rows: values.map(value => [value])+ });+ return table;+}+describe('column edit payloads', () => {test('rename keeps values and records the old name', () => {const payload = buildRenameColumnData(makeTable(), 'region', 'area');@@ -82,4 +105,158 @@ describe('column edit payloads', () => {});expect(buildDeleteColumnData(oneColumn, 'region')).toBeNull();});++ test('type change converts every value and updates field metadata', () => {+ const table = makeColumnTable(['10', '-2', null], ALL_FIELD_TYPES.string, 'STRING');+ table.fields[0].displayFormat = '.2f';++ const result = buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.integer);+ expect(result?.payload?.data.rows).toEqual([[10], [-2], [null]]);+ expect(result?.payload?.data.fields[0]).toMatchObject({+ name: 'value',+ type: ALL_FIELD_TYPES.integer,+ analyzerType: 'INT',+ format: ''+ });+ expect(+ (result?.payload?.data.fields[0] as {displayFormat?: string} | undefined)?.displayFormat+ ).toBeUndefined();+ });++ test('type change reports the first invalid row without mutating the table', () => {+ const table = makeColumnTable(['10', 'not-a-number', '30'], ALL_FIELD_TYPES.string, 'STRING');++ const result = buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.integer);+ expect(result).toEqual({+ error: {rowIndex: 1, value: 'not-a-number', reason: 'is not a safe integer'}+ });+ expect(table.fields[0].type).toBe(ALL_FIELD_TYPES.string);+ expect(table.dataContainer.rowAsArray(0)[0]).toBe('10');+ expect(getCompatibleColumnTypes(table.dataContainer, 0, ALL_FIELD_TYPES.string)).toEqual([+ ALL_FIELD_TYPES.string+ ]);+ expect(result?.error && formatColumnTypeConversionError(result.error)).toBe(+ 'row 2 value “not-a-number” is not a safe integer'+ );+ });++ test('type change parses timestamps and rejects types outside the initial scope', () => {+ const table = makeColumnTable(+ ['2024-01-01T12:30:00Z', '2025-02-02T01:02:03Z'],+ ALL_FIELD_TYPES.string,+ 'STRING'+ );++ const timestamp = buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.timestamp);+ expect(timestamp?.payload?.data.fields[0]).toMatchObject({+ type: ALL_FIELD_TYPES.timestamp,+ analyzerType: 'DATETIME',+ format: 'YYYY-M-DTH:m:s'+ });++ expect(buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.boolean)).toBeNull();+ expect(buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.array)).toBeNull();+ });++ test.each([+ {type: ALL_FIELD_TYPES.integer, values: [1_700_000_000, 1_700_000_001]},+ {type: ALL_FIELD_TYPES.real, values: [1_700_000_000.4, 1_700_000_001.6]},+ {type: ALL_FIELD_TYPES.string, values: ['1700000000.4', '1700000001.6']}+ ])('timestamp conversion preserves plausible $type epochs without rounding', ({type, values}) => {+ const table = makeColumnTable(values, type);+ const converted = buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.timestamp);+ expect(converted?.payload?.data.rows).toEqual(+ type === ALL_FIELD_TYPES.integer+ ? [[1_700_000_000], [1_700_000_001]]+ : [[1_700_000_000.4], [1_700_000_001.6]]+ );+ expect(converted?.payload?.data.fields[0]).toMatchObject({+ type: ALL_FIELD_TYPES.timestamp,+ analyzerType: 'TIME',+ format: 'x'+ });+ expect(getCompatibleColumnTypes(table.dataContainer, 0, type)).toContain(+ ALL_FIELD_TYPES.timestamp+ );+ });++ test('timestamp conversion rejects measurements', () => {+ const measurements = makeColumnTable([3.1, 4], ALL_FIELD_TYPES.real);+ expect(buildChangeColumnTypeData(measurements, 'value', ALL_FIELD_TYPES.timestamp)).toEqual({+ error: {+ rowIndex: 0,+ value: 3.1,+ reason: 'is not a plausible epoch timestamp'+ }+ });+ expect(getCompatibleColumnTypes(measurements.dataContainer, 0, ALL_FIELD_TYPES.real)).toEqual([+ ALL_FIELD_TYPES.real,+ ALL_FIELD_TYPES.string+ ]);+ });++ test('type changes omit numeric conversions that do not change column behavior', () => {+ const integers = makeColumnTable([1_700_000_000, 1_700_000_001], ALL_FIELD_TYPES.integer);+ expect(getCompatibleColumnTypes(integers.dataContainer, 0, ALL_FIELD_TYPES.integer)).toEqual([+ ALL_FIELD_TYPES.integer,+ ALL_FIELD_TYPES.string,+ ALL_FIELD_TYPES.timestamp+ ]);+ expect(buildChangeColumnTypeData(integers, 'value', ALL_FIELD_TYPES.real)).toBeNull();++ const timestamp = buildChangeColumnTypeData(integers, 'value', ALL_FIELD_TYPES.timestamp);+ expect(timestamp?.payload).toBeTruthy();+ if (!timestamp?.payload) {+ throw new Error('Expected timestamp conversion to succeed');+ }+ const timestamps = makeColumnTable([]);+ timestamps.updateSchema(timestamp.payload.data);+ expect(+ getCompatibleColumnTypes(timestamps.dataContainer, 0, ALL_FIELD_TYPES.timestamp)+ ).toEqual([ALL_FIELD_TYPES.integer, ALL_FIELD_TYPES.string, ALL_FIELD_TYPES.timestamp]);+ expect(buildChangeColumnTypeData(timestamps, 'value', ALL_FIELD_TYPES.real)).toBeNull();+ });++ test('real conversion does not silently round large integer strings', () => {+ for (const value of [+ '9007199254740993',+ '9007199254740993.0',+ '9007199254740993e0',+ '9.007199254740993e15'+ ]) {+ const table = makeColumnTable([value]);++ expect(buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.real)).toEqual({+ error: {+ rowIndex: 0,+ value,+ reason: 'loses integer precision as a real number'+ }+ });+ expect(getCompatibleColumnTypes(table.dataContainer, 0, ALL_FIELD_TYPES.string)).toEqual([+ ALL_FIELD_TYPES.string+ ]);+ }+ });++ test('type change preserves untouched Arrow vectors', () => {+ const table = new KeplerTable({+ info: {id: 'arrow', label: 'arrow', type: 'local'},+ color: [0, 0, 0]+ });+ table.updateSchema({+ fields: [+ {name: 'value', type: ALL_FIELD_TYPES.string},+ {name: 'label', type: ALL_FIELD_TYPES.string}+ ],+ rows: [],+ cols: [createArrowVector(['10', '20']), createArrowVector(['a', 'b'])]+ });+ const untouched = table.dataContainer.getColumn?.(1);++ const result = buildChangeColumnTypeData(table, 'value', ALL_FIELD_TYPES.integer);+ expect(result?.payload?.data.rows).toEqual([]);+ expect(result?.payload?.data.cols?.[1]).toBe(untouched);+ expect(Array.from(result?.payload?.data.cols?.[0] as Iterable<unknown>)).toEqual([10, 20]);+ });});diff --git a/src/table/src/dataset-ops/column-edit.ts b/src/table/src/dataset-ops/column-edit.tsindex facfb6e2c..287d9e1ef 100644--- a/src/table/src/dataset-ops/column-edit.ts+++ b/src/table/src/dataset-ops/column-edit.ts@@ -1,8 +1,10 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project+import {Analyzer, DATA_TYPES as AnalyzerDataTypes} from 'type-analyzer';+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';import {Field, ProtoDataset} from '@kepler.gl/types';-import {DataContainerInterface} from '@kepler.gl/utils';+import {createArrowVector, DataContainerInterface} from '@kepler.gl/utils';export type ColumnEditDataset = {fields: Field[];@@ -14,6 +16,279 @@ export type ColumnEditPayload = {renames: Record<string, string>;};+export const EDITABLE_FIELD_TYPES = [+ ALL_FIELD_TYPES.integer,+ ALL_FIELD_TYPES.real,+ ALL_FIELD_TYPES.string,+ ALL_FIELD_TYPES.timestamp+] as const;++export type EditableFieldType = (typeof EDITABLE_FIELD_TYPES)[number];++const EDITABLE_TARGET_TYPES: Record<EditableFieldType, readonly EditableFieldType[]> = {+ [ALL_FIELD_TYPES.integer]: [ALL_FIELD_TYPES.string, ALL_FIELD_TYPES.timestamp],+ [ALL_FIELD_TYPES.real]: [ALL_FIELD_TYPES.string, ALL_FIELD_TYPES.timestamp],+ [ALL_FIELD_TYPES.string]: [+ ALL_FIELD_TYPES.integer,+ ALL_FIELD_TYPES.real,+ ALL_FIELD_TYPES.timestamp+ ],+ [ALL_FIELD_TYPES.timestamp]: [ALL_FIELD_TYPES.integer, ALL_FIELD_TYPES.string]+};++function isEditableFieldType(type: string): type is EditableFieldType {+ return (EDITABLE_FIELD_TYPES as readonly string[]).includes(type);+}++export type ColumnTypeConversionError = {+ rowIndex: number;+ value: unknown;+ reason: string;+};++export type ChangeColumnTypeResult =+ | {payload: ColumnEditPayload; error?: never}+ | {payload?: never; error: ColumnTypeConversionError};++function columnValuePreview(value: unknown): string {+ let preview: string;+ try {+ preview = typeof value === 'string' ? value : JSON.stringify(value);+ } catch {+ preview = String(value);+ }+ if (preview === undefined) {+ preview = String(value);+ }+ return preview.length > 80 ? `${preview.slice(0, 77)}...` : preview;+}++export function formatColumnTypeConversionError(error: ColumnTypeConversionError): string {+ return `row ${error.rowIndex + 1} value “${columnValuePreview(error.value)}” ${error.reason}`;+}++const DECIMAL_NUMBER = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/i;+const INTEGER = /^[+-]?\d+$/;+const MIN_ABSOLUTE_EPOCH = 100_000_000;+const MILLISECOND_EPOCH_THRESHOLD = 100_000_000_000;+const MAX_ABSOLUTE_EPOCH_MILLISECONDS = 9_999_999_999_999;++const ANALYZER_TYPE_BY_FIELD_TYPE: Record<string, string> = {+ [ALL_FIELD_TYPES.integer]: AnalyzerDataTypes.INT,+ [ALL_FIELD_TYPES.real]: AnalyzerDataTypes.FLOAT,+ [ALL_FIELD_TYPES.string]: AnalyzerDataTypes.STRING,+ [ALL_FIELD_TYPES.timestamp]: AnalyzerDataTypes.DATETIME+};++function analyzerMetadata(value: unknown): {type: string; format: string} | null {+ const metadata = Analyzer.computeColMeta([{value}])?.[0];+ return metadata ? {type: metadata.type, format: metadata.format || ''} : null;+}++type ColumnConversionState = {+ analyzerType: string;+ format: string;+};++function conversionState(type: EditableFieldType): ColumnConversionState {+ return {+ analyzerType: type === ALL_FIELD_TYPES.timestamp ? '' : ANALYZER_TYPE_BY_FIELD_TYPE[type],+ format: ''+ };+}++function numericValue(value: unknown, integerOnly: boolean): number {+ if (typeof value === 'number') {+ return value;+ }+ if (typeof value === 'bigint') {+ return Number(value);+ }+ if (typeof value !== 'string') {+ return Number.NaN;+ }+ const trimmed = value.trim();+ if (!(integerOnly ? INTEGER : DECIMAL_NUMBER).test(trimmed)) {+ return Number.NaN;+ }+ return Number(trimmed);+}++function exactIntegerFromDecimal(value: string): bigint | null {+ const trimmed = value.trim().toLowerCase();+ if (!DECIMAL_NUMBER.test(trimmed)) {+ return null;+ }++ const unsigned = trimmed.replace(/^[+-]/, '');+ const [mantissa, exponentString] = unsigned.split('e');+ const [whole, fraction = ''] = mantissa.split('.');+ const digits = `${whole || '0'}${fraction}`;+ if (BigInt(digits) === 0n) {+ return 0n;+ }+ const exponent = exponentString ? Number(exponentString) : 0;+ if (!Number.isSafeInteger(exponent)) {+ return null;+ }++ const decimalPlaces = fraction.length - exponent;+ let integerDigits: string;+ if (decimalPlaces <= 0) {+ integerDigits = `${digits}${'0'.repeat(-decimalPlaces)}`;+ } else {+ const fractionalTail = digits.slice(-decimalPlaces);+ if (decimalPlaces >= digits.length || /[1-9]/.test(fractionalTail)) {+ return null;+ }+ integerDigits = digits.slice(0, -decimalPlaces);+ }+ const integer = BigInt(integerDigits);+ return trimmed.startsWith('-') ? -integer : integer;+}++function losesIntegerPrecision(value: unknown, parsed: number): boolean {+ if (!Number.isFinite(parsed) || !Number.isInteger(parsed)) {+ return false;+ }+ if (typeof value === 'bigint') {+ return BigInt(parsed) !== value;+ }+ if (typeof value === 'string') {+ const exactInteger = exactIntegerFromDecimal(value);+ return exactInteger !== null && BigInt(parsed) !== exactInteger;+ }+ return false;+}++function convertColumnValue(+ value: unknown,+ type: EditableFieldType,+ rowIndex: number,+ state: ColumnConversionState,+ output?: unknown[]+): ColumnTypeConversionError | null {+ if (value === null || value === undefined) {+ output?.push(null);+ return null;+ }++ switch (type) {+ case ALL_FIELD_TYPES.string: {+ try {+ const serialized =+ value instanceof Date+ ? value.toISOString()+ : typeof value === 'object'+ ? JSON.stringify(value)+ : String(value);+ output?.push(serialized ?? String(value));+ return null;+ } catch {+ return {rowIndex, value, reason: 'cannot be serialized as a string'};+ }+ }+ case ALL_FIELD_TYPES.integer:+ case ALL_FIELD_TYPES.real: {+ const integerOnly = type === ALL_FIELD_TYPES.integer;+ const parsed = numericValue(value, integerOnly);+ if (integerOnly ? !Number.isSafeInteger(parsed) : !Number.isFinite(parsed)) {+ return {+ rowIndex,+ value,+ reason: integerOnly ? 'is not a safe integer' : 'is not a finite number'+ };+ }+ if (!integerOnly && losesIntegerPrecision(value, parsed)) {+ return {rowIndex, value, reason: 'loses integer precision as a real number'};+ }+ output?.push(parsed);+ return null;+ }+ case ALL_FIELD_TYPES.timestamp: {+ const source = value instanceof Date ? value.toISOString() : value;+ const isNumeric =+ typeof source === 'number' ||+ typeof source === 'bigint' ||+ (typeof source === 'string' && DECIMAL_NUMBER.test(source.trim()));+ if (isNumeric) {+ const parsed = numericValue(source, false);+ const numericFormat = Math.abs(parsed) >= MILLISECOND_EPOCH_THRESHOLD ? 'X' : 'x';+ const milliseconds = numericFormat === 'x' ? parsed * 1000 : parsed;+ if (+ !Number.isFinite(parsed) ||+ Math.abs(parsed) < MIN_ABSOLUTE_EPOCH ||+ Math.abs(milliseconds) > MAX_ABSOLUTE_EPOCH_MILLISECONDS+ ) {+ return {rowIndex, value, reason: 'is not a plausible epoch timestamp'};+ }+ if (state.format && state.format !== numericFormat) {+ return {rowIndex, value, reason: 'uses a different timestamp format'};+ }+ state.analyzerType = AnalyzerDataTypes.TIME;+ state.format = numericFormat;+ output?.push(parsed);+ return null;+ }++ if (typeof source !== 'string') {+ return {rowIndex, value, reason: 'is not a timestamp'};+ }+ const metadata = analyzerMetadata(source);+ if (+ !metadata ||+ ![AnalyzerDataTypes.TIME, AnalyzerDataTypes.DATETIME].includes(metadata.type)+ ) {+ return {rowIndex, value, reason: 'is not a valid timestamp'};+ }+ if (+ (state.format && state.format !== metadata.format) ||+ (state.analyzerType && state.analyzerType !== metadata.type)+ ) {+ return {rowIndex, value, reason: 'uses a different timestamp format'};+ }+ state.analyzerType = metadata.type;+ state.format = metadata.format;+ output?.push(source);+ return null;+ }+ default:+ return {rowIndex, value, reason: `cannot be converted to ${type}`};+ }+}++/** Validate every candidate in one pass without allocating converted columns. */+export function getCompatibleColumnTypes(+ dataContainer: DataContainerInterface,+ fieldIndex: number,+ currentType: string+): EditableFieldType[] {+ if (+ fieldIndex < 0 ||+ fieldIndex >= dataContainer.numColumns() ||+ !isEditableFieldType(currentType)+ ) {+ return [];+ }++ const candidates = new Map<EditableFieldType, ColumnConversionState>();+ for (const type of EDITABLE_TARGET_TYPES[currentType]) {+ candidates.set(type, conversionState(type));+ }++ const rowCount = dataContainer.numRows();+ for (let rowIndex = 0; rowIndex < rowCount && candidates.size; rowIndex++) {+ const value = dataContainer.valueAt(rowIndex, fieldIndex);+ for (const [type, state] of candidates) {+ if (convertColumnValue(value, type, rowIndex, state)) {+ candidates.delete(type);+ }+ }+ }++ return EDITABLE_FIELD_TYPES.filter(type => type === currentType || candidates.has(type));+}+/*** Field descriptor for `updateDataset`, with accessors and per-column stats* stripped so the table rebuilds them against the new container.@@ -107,3 +382,76 @@ export function buildDeleteColumnData(renames: {}};}++/**+ * Convert an entire column before returning an update payload. A failed value+ * returns its row and leaves the source dataset untouched.+ */+export function buildChangeColumnTypeData(+ dataset: ColumnEditDataset,+ fieldName: string,+ newType: string+): ChangeColumnTypeResult | null {+ if (!isEditableFieldType(newType)) {+ return null;+ }+ const fieldIndex = dataset.fields.findIndex(field => field.name === fieldName);+ const currentType = dataset.fields[fieldIndex]?.type;+ if (+ fieldIndex < 0 ||+ !isEditableFieldType(currentType) ||+ !EDITABLE_TARGET_TYPES[currentType].includes(newType)+ ) {+ return null;+ }++ const state = conversionState(newType);+ const values: unknown[] = [];+ const rowCount = dataset.dataContainer.numRows();+ for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {+ const error = convertColumnValue(+ dataset.dataContainer.valueAt(rowIndex, fieldIndex),+ newType,+ rowIndex,+ state,+ values+ );+ if (error) {+ return {error};+ }+ }++ const fields = dataset.fields.map((field, index) => {+ if (index !== fieldIndex) {+ return schemaField(field, field.name);+ }+ const {+ displayFormat: _displayFormat,+ indexBy: _indexBy,+ ...fieldWithoutTypeSpecificMetadata+ } = field;+ return {+ ...schemaField(fieldWithoutTypeSpecificMetadata, field.name),+ type: newType,+ analyzerType: state.analyzerType || ANALYZER_TYPE_BY_FIELD_TYPE[newType],+ format: state.format+ };+ });++ if (typeof dataset.dataContainer.getColumn === 'function') {+ const cols = dataset.fields.map((_, index) => dataset.dataContainer.getColumn?.(index));+ if (cols.every(Boolean)) {+ cols[fieldIndex] = createArrowVector(values);+ return {payload: {data: {fields, rows: [], cols}, renames: {}}};+ }+ }++ const rows: unknown[][] = new Array(rowCount);+ for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {+ const row = Array.from(dataset.dataContainer.rowAsArray(rowIndex));+ row[fieldIndex] = values[rowIndex];+ rows[rowIndex] = row;+ }++ return {payload: {data: {fields, rows}, renames: {}}};+}diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.tsindex 245cc732e..6de64ebdd 100644--- a/src/utils/src/arrow-data-container.ts+++ b/src/utils/src/arrow-data-container.ts@@ -62,6 +62,11 @@ export function isArrowVector(data: any): data is arrow.Vector {);}+/** Build an Arrow vector while letting Arrow infer the physical type from converted values. */+export function createArrowVector(values: unknown[]): arrow.Vector {+ return arrow.vectorFromArray(values as any[]);+}+/*** Check if data is an Arrow FixedSizeList DataType.* Uses duck-typing instead of `instanceof` to handle DuckDB's bundled Arrow version.diff --git a/src/utils/src/index.ts b/src/utils/src/index.tsindex be976faf6..62fa317e8 100644--- a/src/utils/src/index.ts+++ b/src/utils/src/index.ts@@ -158,6 +158,7 @@ export * from './map-utils';export {ArrowDataContainer,+ createArrowVector,compactArrowTable,arrowDataTypeToAnalyzerDataType,arrowDataTypeToFieldType,