Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix(export): preserve nested Int64 values and report backup failures (#3973)
Igor Dykhtaf5bd66e93d28
diff --git a/src/components/src/modal-container.spec.ts b/src/components/src/modal-container.spec.tsindex 96178f0a9..3260695a2 100644--- a/src/components/src/modal-container.spec.ts+++ b/src/components/src/modal-container.spec.ts@@ -12,12 +12,14 @@ import {EXPORT_DATA_ID,EXPORT_MAP_ID,FIELD_COLORS,- FIELD_TYPE_DISPLAY+ FIELD_TYPE_DISPLAY,+ SAVE_MAP_ID} from '@kepler.gl/constants';import {KeplerTable, parseVectorMetadata} from '@kepler.gl/table';import {KeplerGlLayers} from '@kepler.gl/layers';import {INITIAL_MAP_STATE,+ INITIAL_UI_STATE,INITIAL_VIS_STATE,mergeFilters,setFilterUpdater@@ -359,3 +361,137 @@ describe('ExportMapModalStyle', () => {unmount();});});++describe('map export failures', () => {+ test('keeps cloud metadata as a draft until schema saving succeeds on retry', () => {+ const failure = new Error('Cannot export circular dataset values');+ const save = jest.fn((): object => {+ throw failure;+ });+ const draft = {title: 'Edited title', description: 'Draft'};+ const exportFileToCloud = jest.fn();+ const addNotification = jest.fn();+ const toggleModal = jest.fn();+ const setMapInfo = jest.fn();+ const dependencies = ModalContainerFactory.deps.map(() => () => null) as unknown as Parameters<+ typeof ModalContainerFactory+ >;+ dependencies[10] = (({onMapInfoDraft, onSetMapInfo, onConfirm}) =>+ React.createElement(+ 'button',+ {+ onClick: () => {+ onMapInfoDraft(draft);+ onSetMapInfo(draft);+ onConfirm({name: 'test-provider'});+ }+ },+ 'Save map'+ )) as any;+ dependencies[9] = (({children}) => children) as any;+ const ModalContainer = ModalContainerFactory(...dependencies);+ renderWithTheme(+ React.createElement(ModalContainer, {+ rootNode: document.body,+ uiState: {...INITIAL_UI_STATE, currentModal: SAVE_MAP_ID},+ visState: {+ ...INITIAL_VIS_STATE,+ schema: {save}+ },+ visStateActions: {setMapInfo},+ uiStateActions: {toggleModal, addNotification},+ providerActions: {exportFileToCloud},+ providerState: {}+ } as any)+ );+ fireEvent.click(screen.getByRole('button', {name: 'Save map'}));+ expect(exportFileToCloud).not.toHaveBeenCalled();+ expect(toggleModal).not.toHaveBeenCalled();+ expect(setMapInfo).not.toHaveBeenCalled();+ expect(addNotification).toHaveBeenCalledWith(+ expect.objectContaining({+ type: 'error',+ topic: 'global',+ message: `Failed to export map: ${failure.message}`+ })+ );+ const saved = {datasets: [], config: {version: 'v1', config: {}}, info: draft};+ save.mockReturnValue(saved);+ fireEvent.click(screen.getByRole('button', {name: 'Save map'}));+ expect(save).toHaveBeenLastCalledWith(+ expect.objectContaining({+ visState: expect.objectContaining({mapInfo: expect.objectContaining(draft)})+ })+ );+ expect(setMapInfo).toHaveBeenCalledTimes(1);+ expect(setMapInfo).toHaveBeenCalledWith(draft);+ expect(exportFileToCloud).toHaveBeenCalledTimes(1);+ expect(exportFileToCloud).toHaveBeenCalledWith(+ expect.objectContaining({mapData: {map: saved, thumbnail: null}, closeModal: true})+ );+ expect(addNotification).toHaveBeenCalledTimes(1);+ });++ test('keeps the modal and title draft on failure, then closes after a successful retry', () => {+ const cycle: any = {};+ cycle.self = cycle;+ const save = jest.fn(() => cycle);+ const addNotification = jest.fn();+ const toggleModal = jest.fn();+ const setMapInfo = jest.fn();+ const download = jest.fn();+ Object.defineProperty(window.navigator, 'msSaveOrOpenBlob', {+ configurable: true,+ value: download+ });+ const dependencies = ModalContainerFactory.deps.map(() => () => null) as unknown as Parameters<+ typeof ModalContainerFactory+ >;+ dependencies[6] = (({onMapInfoDraft}) =>+ React.createElement(+ 'button',+ {+ onClick: () => onMapInfoDraft({title: 'Edited title', description: 'Draft'})+ },+ 'Edit title'+ )) as any;+ dependencies[9] = (({onConfirm, children}) =>+ React.createElement(+ 'div',+ null,+ children,+ React.createElement('button', {onClick: onConfirm}, 'Export map')+ )) as any;+ const ModalContainer = ModalContainerFactory(...dependencies);+ renderWithTheme(+ React.createElement(ModalContainer, {+ rootNode: document.body,+ uiState: {+ ...INITIAL_UI_STATE,+ currentModal: EXPORT_MAP_ID,+ exportMap: {...INITIAL_UI_STATE.exportMap, format: 'json', json: {hasData: true}}+ },+ visState: {...INITIAL_VIS_STATE, schema: {getConfigToSave: () => ({}), save}},+ visStateActions: {setMapInfo},+ uiStateActions: {toggleModal, addNotification}+ } as any)+ );+ fireEvent.click(screen.getByRole('button', {name: 'Edit title'}));+ fireEvent.click(screen.getByRole('button', {name: 'Export map'}));+ expect(download).not.toHaveBeenCalled();+ expect(toggleModal).not.toHaveBeenCalled();+ expect(setMapInfo).not.toHaveBeenCalled();+ expect(addNotification).toHaveBeenCalledWith(+ expect.objectContaining({+ type: 'error',+ topic: 'global',+ message: expect.stringContaining('Failed to export map:')+ })+ );+ save.mockReturnValue({datasets: [], config: {version: 'v1', config: {}}, info: {}});+ fireEvent.click(screen.getByRole('button', {name: 'Export map'}));+ expect(download).toHaveBeenCalledTimes(1);+ expect(setMapInfo).toHaveBeenCalledWith({title: 'Edited title', description: 'Draft'});+ expect(toggleModal).toHaveBeenCalledWith(null);+ });+});diff --git a/src/components/src/modal-container.tsx b/src/components/src/modal-container.tsxindex 022ca9eda..74ef6c0c7 100644--- a/src/components/src/modal-container.tsx+++ b/src/components/src/modal-container.tsx@@ -347,28 +347,48 @@ export default function ModalContainerFactory(});};+ _onExportMapError = (error: unknown) => {+ const detail = error instanceof Error ? error.message : String(error);+ this.props.uiStateActions.addNotification({+ id: 'EXPORT_MAP_ERROR_ID',+ type: 'error',+ topic: 'global',+ message: `Failed to export map: ${detail}`+ });+ };+_onExportMap = () => {const {uiState} = this._propsWithMapInfoDraft();const {format, includeLayerApiKeys, fileName, appName, appWebsite, appIcon, appNameColor} =uiState.exportMap;- (format === EXPORT_MAP_FORMATS.HTML ? exportHtml : exportJson)(- this._propsWithMapInfoDraft(),- {- ...(uiState.exportMap[format] || {}),- includeLayerApiKeys,- fileName,- appName,- appWebsite,- appIcon,- appNameColor- }- );- this._commitMapInfoDraft();- this._closeModal();+ try {+ (format === EXPORT_MAP_FORMATS.HTML ? exportHtml : exportJson)(+ this._propsWithMapInfoDraft(),+ {+ ...(uiState.exportMap[format] || {}),+ includeLayerApiKeys,+ fileName,+ appName,+ appWebsite,+ appIcon,+ appNameColor+ }+ );+ this._commitMapInfoDraft();+ this._closeModal();+ } catch (error) {+ this._onExportMapError(error);+ }};_exportFileToCloud = ({provider, isPublic, overwrite, closeModal}) => {- const toSave = exportMap(this._propsWithMapInfoDraft());+ let toSave;+ try {+ toSave = exportMap(this._propsWithMapInfoDraft());+ } catch (error) {+ this._onExportMapError(error);+ return;+ }this._commitMapInfoDraft();this.props.providerActions.exportFileToCloud({@@ -677,7 +697,7 @@ export default function ModalContainerFactory(fileName={uiState.exportMap.fileName}onChangeFileName={uiStateActions.setExportMapFileName}onMapInfoDraft={this._onMapInfoDraft}- onSetMapInfo={visStateActions.setMapInfo}+ onSetMapInfo={this._onMapInfoDraft}cleanupExportImage={uiStateActions.cleanupExportImage}onUpdateImageSetting={uiStateActions.setExportImageSetting}onCancel={this._closeModal}diff --git a/src/components/src/modals/save-map-modal.tsx b/src/components/src/modals/save-map-modal.tsxindex 59166b2e6..cca089834 100644--- a/src/components/src/modals/save-map-modal.tsx+++ b/src/components/src/modals/save-map-modal.tsx@@ -82,7 +82,7 @@ type SaveMapModalProps = {// callbacksonUpdateImageSetting: ImageModalContainerProps['onUpdateImageSetting'];cleanupExportImage: typeof cleanupExportImageAction;- onSetMapInfo: typeof setMapInfo;+ onSetMapInfo: (info: Parameters<typeof setMapInfo>[0]) => void;onConfirm: (provider: Provider) => void;onCancel: () => void;};diff --git a/src/schemas/src/dataset-schema.ts b/src/schemas/src/dataset-schema.tsindex 54017af31..72e98433d 100644--- a/src/schemas/src/dataset-schema.ts+++ b/src/schemas/src/dataset-schema.ts@@ -2,6 +2,7 @@// Copyright contributors to the kepler.gl projectimport pick from 'es-toolkit/compat/pick';+import isPlainObject from 'es-toolkit/compat/isPlainObject';import {console as globalConsole} from 'global/window';import * as arrow from 'apache-arrow';@@ -107,6 +108,47 @@ export const propertiesV1 = {disableDataOperation: null};+/** Convert nested Arrow lists/structs and BigInts without changing live values. */+function getValueForSaving(value: any, ancestors = new Set<object>()): any {+ if (typeof value === 'bigint') {+ const number = Number(value);+ return Number.isSafeInteger(number) ? number : value.toString();+ }+ // Leave custom JSON representations intact; Arrow wrappers need normalization below.+ if (+ typeof value?.toJSON === 'function' &&+ !(value instanceof arrow.StructRow) &&+ !(value instanceof arrow.Vector)+ ) {+ return value;+ }+ const plain = value instanceof arrow.StructRow ? value.toJSON() : value;+ const isList = Array.isArray(plain) || plain instanceof arrow.Vector;+ if (!isList && !isPlainObject(plain)) return value;+ if (ancestors.has(value)) throw new TypeError('Cannot export circular dataset values');+ ancestors.add(value);+ let saved = plain;+ if (isList) {+ const items = plain instanceof arrow.Vector ? Array.from(plain) : plain;+ saved = items;+ for (let i = 0; i < items.length; i++) {+ const next = getValueForSaving(items[i], ancestors);+ if (next === items[i]) continue;+ if (saved === items) saved = items.slice();+ saved[i] = next;+ }+ } else {+ for (const key of Object.keys(plain)) {+ const next = getValueForSaving(plain[key], ancestors);+ if (next === plain[key]) continue;+ if (saved === plain) saved = {...plain};+ saved[key] = next;+ }+ }+ ancestors.delete(value);+ return saved;+}+/*** TODO Consider moving this cast to ArrowDataContainer?* Prepare a data container for export as part of json / html files.@@ -135,14 +177,6 @@ const getAllDataForSaving = (dataContainer: DataContainerInterface): any[][] =>const value = row[columnIndex];row[columnIndex] = value == null ? null : new Date(value).toISOString();});- } else if (arrow.DataType.isInt(column.type) && column.type.bitWidth === 64) {- allData.forEach(row => {- const value = row[columnIndex];- if (typeof value === 'bigint') {- const number = Number(value);- row[columnIndex] = Number.isSafeInteger(number) ? number : value.toString();- }- });} else if (field?.type === ALL_FIELD_TYPES.geoarrow) {const formatter = FIELD_DISPLAY_FORMAT[ALL_FIELD_TYPES.geoarrow];allData.forEach(row => {@@ -150,27 +184,10 @@ const getAllDataForSaving = (dataContainer: DataContainerInterface): any[][] =>});}}- } else {- // Row containers expose their live backing rows. Copy only when conversion- // is needed, leaving other rows and exact stored values intact.- let savedData = allData;- for (let rowIndex = 0; rowIndex < allData.length; rowIndex++) {- const row = allData[rowIndex];- if (row == null) continue;- let savedRow = row;- for (let columnIndex = 0; columnIndex < row.length; columnIndex++) {- const value = row[columnIndex];- if (typeof value !== 'bigint') continue;- if (savedData === allData) savedData = allData.slice();- if (savedRow === row) savedRow = savedData[rowIndex] = row.slice();- const number = Number(value);- savedRow[columnIndex] = Number.isSafeInteger(number) ? number : value.toString();- }- }- return savedData;}- return allData;+ // Row containers expose live rows: copy only values that need conversion.+ return getValueForSaving(allData);};/**diff --git a/src/utils/src/export-utils.spec.ts b/src/utils/src/export-utils.spec.tsindex dc1e099d3..b1089c5f7 100644--- a/src/utils/src/export-utils.spec.ts+++ b/src/utils/src/export-utils.spec.ts@@ -1,7 +1,7 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import {exportJson} from './export-utils';+import {exportJson, exportToJsonString} from './export-utils';function readJsonBlob(blob: Blob): Promise<unknown> {return new Promise((resolve, reject) => {@@ -76,3 +76,28 @@ describe('exportJson', () => {});});});++describe('JSON export failures', () => {+ test('propagates serialization failures and never downloads an error-only file', () => {+ const cycle: any = {};+ cycle.self = cycle;+ const saveBlob = jest.fn();+ Object.defineProperty(window.navigator, 'msSaveOrOpenBlob', {+ configurable: true,+ value: saveBlob+ });+ expect(() => exportToJsonString(cycle)).toThrow(TypeError);+ expect(() => exportJson({visState: {schema: {save: () => cycle}}}, {hasData: true})).toThrow(+ TypeError+ );+ expect(saveBlob).not.toHaveBeenCalled();+ const failure = new Error('Cannot read export data');+ expect(() =>+ exportToJsonString({+ toJSON() {+ throw failure;+ }+ })+ ).toThrow(failure);+ });+});diff --git a/src/utils/src/export-utils.ts b/src/utils/src/export-utils.tsindex e488f8233..a26c52b39 100644--- a/src/utils/src/export-utils.ts+++ b/src/utils/src/export-utils.ts@@ -167,13 +167,7 @@ export function exportImage(uiStateExportImage: ExportImage, filename?: string)}export function exportToJsonString(data) {- try {- return JSON.stringify(data);- } catch (e) {- if (e instanceof TypeError) return e.message;- // Non-Standard Error Object Property- return (e as any).description;- }+ return JSON.stringify(data);}/** Dataset metadata fields that hold a private layer API key. */diff --git a/test/node/schemas/dataset-schema-test.js b/test/node/schemas/dataset-schema-test.jsindex bf8c29ab4..ee95f63e9 100644--- a/test/node/schemas/dataset-schema-test.js+++ b/test/node/schemas/dataset-schema-test.js@@ -345,6 +345,162 @@ test('#DatasetSchema -> Int64 values survive JSON, HTML and reload without precit.end();});+test('#DatasetSchema -> nested Arrow Int64 values survive a complete JSON backup and reload', t => {+ const item = new arrow.Struct([+ new arrow.Field('signed', new arrow.Int64(), true),+ new arrow.Field('unsigned', new arrow.Uint64(), true)+ ]);+ const type = new arrow.Struct([+ new arrow.Field('items', new arrow.List(new arrow.Field('item', item, true)), true)+ ]);+ const values = [+ {+ items: [+ {signed: 42n, unsigned: 9007199254740991n},+ {signed: -9223372036854775808n, unsigned: 18446744073709551615n},+ {signed: null, unsigned: 0n},+ null+ ]+ },+ {items: []},+ {items: null},+ null+ ];+ const source = new arrow.Table({nested: arrow.vectorFromArray(values, type)});+ const ipc = arrow.tableFromIPC(arrow.tableToIPC(source, 'file'));+ const dataset = new KeplerTable({info: {id: 'nested'}, color: [1, 2, 3]}).updateSchema(+ processArrowTable({shape: 'arrow-table', data: ipc})+ );+ const expected = [+ [+ {+ items: [+ {signed: 42, unsigned: 9007199254740991},+ {signed: '-9223372036854775808', unsigned: '18446744073709551615'},+ {signed: null, unsigned: 0},+ null+ ]+ }+ ],+ [{items: []}],+ [{items: null}],+ [null]+ ];+ const saved = datasetSchema[VERSIONS.v1].save(dataset);+ t.doesNotThrow(() => {+ const payload = JSON.parse(+ exportToJsonString({+ datasets: [{version: 'v1', data: saved}],+ config: {version: 'v1', config: {}}+ })+ );+ t.deepEqual(payload.datasets[0].data.allData, expected, 'full backup has exact nested values');+ const reloaded = reloadDataset(payload.datasets[0].data);+ t.deepEqual(reloaded.dataContainer.flattenData(), expected, 'reload retains nesting and nulls');+ t.deepEqual(datasetSchema[VERSIONS.v1].save(reloaded).allData, expected, 'resave is stable');+ const html = exportMapToHTML(payload);+ t.deepEqual(+ JSON.parse(html.match(/const datasets = (.*);/)[1]),+ payload.datasets,+ 'HTML uses the same saved values'+ );+ }, 'backup is valid JSON and reloads');+ t.equal(+ dataset.dataContainer.valueAt(0, 0).items.get(1).signed,+ -9223372036854775808n,+ 'live Arrow values are not mutated'+ );+ t.equal(+ ipc.getChild('nested').get(0).items.get(1).unsigned,+ 18446744073709551615n,+ 'source buffers are not mutated'+ );++ const rows = [[{items: [1n, null, {value: 9007199254740993n}]}]];+ const rowTable = new KeplerTable({info: {id: 'nested-rows'}, color: [1, 2, 3]}).updateSchema({+ fields: [{name: 'nested', type: 'object'}],+ rows+ });+ t.deepEqual(+ datasetSchema[VERSIONS.v1].save(rowTable).allData,+ [[{items: [1, null, {value: '9007199254740993'}]}]],+ 'row-backed nested values use the same policy'+ );+ t.equal(rows[0][0].items[2].value, 9007199254740993n, 'live row-backed objects are not mutated');+ t.end();+});++test('#DatasetSchema -> nested custom JSON serializers retain their representations', t => {+ class Identifier {+ constructor() {+ this.raw = 9007199254740993n;+ }+ toJSON() {+ return {id: this.raw.toString()};+ }+ }+ class CachedValue {+ constructor() {+ this.cache = {owner: this};+ }+ toJSON() {+ return {value: 'valid'};+ }+ }+ const custom = {+ raw: 42n,+ toJSON() {+ return {value: this.raw.toString()};+ }+ };+ custom.cache = custom;+ const identifier = new Identifier();+ const cached = new CachedValue();+ const rows = [[{identifier, cached, custom, ordinary: {value: 42n}}]];+ const dataset = new KeplerTable({info: {id: 'custom-json'}, color: [1, 2, 3]}).updateSchema({+ fields: [{name: 'nested', type: 'object'}],+ rows+ });++ const saved = datasetSchema[VERSIONS.v1].save(dataset);+ const expected = [+ [+ {+ identifier: {id: '9007199254740993'},+ cached: {value: 'valid'},+ custom: {value: '42'},+ ordinary: {value: 42}+ }+ ]+ ];+ t.deepEqual(+ JSON.parse(exportToJsonString(saved)).allData,+ expected,+ 'preserves class and plain-object serializers while normalizing ordinary nested values'+ );+ t.deepEqual(reloadDataset(saved).dataContainer.flattenData(), expected, 'custom values reload');+ t.equal(saved.allData[0][0].identifier, identifier, 'retains the custom class instance');+ t.equal(saved.allData[0][0].cached, cached, 'does not traverse the cyclic internal cache');+ t.equal(identifier.raw, 9007199254740993n, 'does not change live class data');+ t.equal(rows[0][0].ordinary.value, 42n, 'does not change ordinary live nested data');+ t.end();+});++test('#DatasetSchema -> rejects ordinary circular values without a JSON serializer', t => {+ const cycle = {};+ cycle.self = cycle;+ const dataset = new KeplerTable({info: {id: 'circular'}, color: [1, 2, 3]}).updateSchema({+ fields: [{name: 'nested', type: 'object'}],+ rows: [[cycle]]+ });+ t.throws(+ () => datasetSchema[VERSIONS.v1].save(dataset),+ /Cannot export circular dataset values/,+ 'still rejects an actual serialization cycle'+ );+ t.end();+});+test('#DatasetSchema -> joined Int64 rows save and reload without changing live rows', t => {const values = [0n,