Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix(export): preserve nested Int64 values and report backup failures (#3973)

Igor Dykhtaf5bd66e93d28
diff --git a/src/components/src/modal-container.spec.ts b/src/components/src/modal-container.spec.ts
index 96178f0a9..3260695a2 100644
--- a/src/components/src/modal-container.spec.ts
+++ b/src/components/src/modal-container.spec.ts
@@ -12,12 +12,14 @@ import {
EXPORT_DATA_ID,
EXPORT_MAP_ID,
FIELD_COLORS,
- FIELD_TYPE_DISPLAY
+ FIELD_TYPE_DISPLAY,
+ SAVE_MAP_ID
} from '@kepler.gl/constants';
import {KeplerTable, parseVectorMetadata} from '@kepler.gl/table';
import {KeplerGlLayers} from '@kepler.gl/layers';
import {
INITIAL_MAP_STATE,
+ INITIAL_UI_STATE,
INITIAL_VIS_STATE,
mergeFilters,
setFilterUpdater
@@ -359,3 +361,137 @@ describe('ExportMapModalStyle', () => {
unmount();
});
});
+
+describe('map export failures', () => {
+ test('keeps cloud metadata as a draft until schema saving succeeds on retry', () => {
+ const failure = new Error('Cannot export circular dataset values');
+ const save = jest.fn((): object => {
+ throw failure;
+ });
+ const draft = {title: 'Edited title', description: 'Draft'};
+ const exportFileToCloud = jest.fn();
+ const addNotification = jest.fn();
+ const toggleModal = jest.fn();
+ const setMapInfo = jest.fn();
+ const dependencies = ModalContainerFactory.deps.map(() => () => null) as unknown as Parameters<
+ typeof ModalContainerFactory
+ >;
+ dependencies[10] = (({onMapInfoDraft, onSetMapInfo, onConfirm}) =>
+ React.createElement(
+ 'button',
+ {
+ onClick: () => {
+ onMapInfoDraft(draft);
+ onSetMapInfo(draft);
+ onConfirm({name: 'test-provider'});
+ }
+ },
+ 'Save map'
+ )) as any;
+ dependencies[9] = (({children}) => children) as any;
+ const ModalContainer = ModalContainerFactory(...dependencies);
+ renderWithTheme(
+ React.createElement(ModalContainer, {
+ rootNode: document.body,
+ uiState: {...INITIAL_UI_STATE, currentModal: SAVE_MAP_ID},
+ visState: {
+ ...INITIAL_VIS_STATE,
+ schema: {save}
+ },
+ visStateActions: {setMapInfo},
+ uiStateActions: {toggleModal, addNotification},
+ providerActions: {exportFileToCloud},
+ providerState: {}
+ } as any)
+ );
+ fireEvent.click(screen.getByRole('button', {name: 'Save map'}));
+ expect(exportFileToCloud).not.toHaveBeenCalled();
+ expect(toggleModal).not.toHaveBeenCalled();
+ expect(setMapInfo).not.toHaveBeenCalled();
+ expect(addNotification).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: 'error',
+ topic: 'global',
+ message: `Failed to export map: ${failure.message}`
+ })
+ );
+ const saved = {datasets: [], config: {version: 'v1', config: {}}, info: draft};
+ save.mockReturnValue(saved);
+ fireEvent.click(screen.getByRole('button', {name: 'Save map'}));
+ expect(save).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ visState: expect.objectContaining({mapInfo: expect.objectContaining(draft)})
+ })
+ );
+ expect(setMapInfo).toHaveBeenCalledTimes(1);
+ expect(setMapInfo).toHaveBeenCalledWith(draft);
+ expect(exportFileToCloud).toHaveBeenCalledTimes(1);
+ expect(exportFileToCloud).toHaveBeenCalledWith(
+ expect.objectContaining({mapData: {map: saved, thumbnail: null}, closeModal: true})
+ );
+ expect(addNotification).toHaveBeenCalledTimes(1);
+ });
+
+ test('keeps the modal and title draft on failure, then closes after a successful retry', () => {
+ const cycle: any = {};
+ cycle.self = cycle;
+ const save = jest.fn(() => cycle);
+ const addNotification = jest.fn();
+ const toggleModal = jest.fn();
+ const setMapInfo = jest.fn();
+ const download = jest.fn();
+ Object.defineProperty(window.navigator, 'msSaveOrOpenBlob', {
+ configurable: true,
+ value: download
+ });
+ const dependencies = ModalContainerFactory.deps.map(() => () => null) as unknown as Parameters<
+ typeof ModalContainerFactory
+ >;
+ dependencies[6] = (({onMapInfoDraft}) =>
+ React.createElement(
+ 'button',
+ {
+ onClick: () => onMapInfoDraft({title: 'Edited title', description: 'Draft'})
+ },
+ 'Edit title'
+ )) as any;
+ dependencies[9] = (({onConfirm, children}) =>
+ React.createElement(
+ 'div',
+ null,
+ children,
+ React.createElement('button', {onClick: onConfirm}, 'Export map')
+ )) as any;
+ const ModalContainer = ModalContainerFactory(...dependencies);
+ renderWithTheme(
+ React.createElement(ModalContainer, {
+ rootNode: document.body,
+ uiState: {
+ ...INITIAL_UI_STATE,
+ currentModal: EXPORT_MAP_ID,
+ exportMap: {...INITIAL_UI_STATE.exportMap, format: 'json', json: {hasData: true}}
+ },
+ visState: {...INITIAL_VIS_STATE, schema: {getConfigToSave: () => ({}), save}},
+ visStateActions: {setMapInfo},
+ uiStateActions: {toggleModal, addNotification}
+ } as any)
+ );
+ fireEvent.click(screen.getByRole('button', {name: 'Edit title'}));
+ fireEvent.click(screen.getByRole('button', {name: 'Export map'}));
+ expect(download).not.toHaveBeenCalled();
+ expect(toggleModal).not.toHaveBeenCalled();
+ expect(setMapInfo).not.toHaveBeenCalled();
+ expect(addNotification).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: 'error',
+ topic: 'global',
+ message: expect.stringContaining('Failed to export map:')
+ })
+ );
+ save.mockReturnValue({datasets: [], config: {version: 'v1', config: {}}, info: {}});
+ fireEvent.click(screen.getByRole('button', {name: 'Export map'}));
+ expect(download).toHaveBeenCalledTimes(1);
+ expect(setMapInfo).toHaveBeenCalledWith({title: 'Edited title', description: 'Draft'});
+ expect(toggleModal).toHaveBeenCalledWith(null);
+ });
+});
diff --git a/src/components/src/modal-container.tsx b/src/components/src/modal-container.tsx
index 022ca9eda..74ef6c0c7 100644
--- a/src/components/src/modal-container.tsx
+++ b/src/components/src/modal-container.tsx
@@ -347,28 +347,48 @@ export default function ModalContainerFactory(
});
};
+ _onExportMapError = (error: unknown) => {
+ const detail = error instanceof Error ? error.message : String(error);
+ this.props.uiStateActions.addNotification({
+ id: 'EXPORT_MAP_ERROR_ID',
+ type: 'error',
+ topic: 'global',
+ message: `Failed to export map: ${detail}`
+ });
+ };
+
_onExportMap = () => {
const {uiState} = this._propsWithMapInfoDraft();
const {format, includeLayerApiKeys, fileName, appName, appWebsite, appIcon, appNameColor} =
uiState.exportMap;
- (format === EXPORT_MAP_FORMATS.HTML ? exportHtml : exportJson)(
- this._propsWithMapInfoDraft(),
- {
- ...(uiState.exportMap[format] || {}),
- includeLayerApiKeys,
- fileName,
- appName,
- appWebsite,
- appIcon,
- appNameColor
- }
- );
- this._commitMapInfoDraft();
- this._closeModal();
+ try {
+ (format === EXPORT_MAP_FORMATS.HTML ? exportHtml : exportJson)(
+ this._propsWithMapInfoDraft(),
+ {
+ ...(uiState.exportMap[format] || {}),
+ includeLayerApiKeys,
+ fileName,
+ appName,
+ appWebsite,
+ appIcon,
+ appNameColor
+ }
+ );
+ this._commitMapInfoDraft();
+ this._closeModal();
+ } catch (error) {
+ this._onExportMapError(error);
+ }
};
_exportFileToCloud = ({provider, isPublic, overwrite, closeModal}) => {
- const toSave = exportMap(this._propsWithMapInfoDraft());
+ let toSave;
+ try {
+ toSave = exportMap(this._propsWithMapInfoDraft());
+ } catch (error) {
+ this._onExportMapError(error);
+ return;
+ }
this._commitMapInfoDraft();
this.props.providerActions.exportFileToCloud({
@@ -677,7 +697,7 @@ export default function ModalContainerFactory(
fileName={uiState.exportMap.fileName}
onChangeFileName={uiStateActions.setExportMapFileName}
onMapInfoDraft={this._onMapInfoDraft}
- onSetMapInfo={visStateActions.setMapInfo}
+ onSetMapInfo={this._onMapInfoDraft}
cleanupExportImage={uiStateActions.cleanupExportImage}
onUpdateImageSetting={uiStateActions.setExportImageSetting}
onCancel={this._closeModal}
diff --git a/src/components/src/modals/save-map-modal.tsx b/src/components/src/modals/save-map-modal.tsx
index 59166b2e6..cca089834 100644
--- a/src/components/src/modals/save-map-modal.tsx
+++ b/src/components/src/modals/save-map-modal.tsx
@@ -82,7 +82,7 @@ type SaveMapModalProps = {
// callbacks
onUpdateImageSetting: ImageModalContainerProps['onUpdateImageSetting'];
cleanupExportImage: typeof cleanupExportImageAction;
- onSetMapInfo: typeof setMapInfo;
+ onSetMapInfo: (info: Parameters<typeof setMapInfo>[0]) => void;
onConfirm: (provider: Provider) => void;
onCancel: () => void;
};
diff --git a/src/schemas/src/dataset-schema.ts b/src/schemas/src/dataset-schema.ts
index 54017af31..72e98433d 100644
--- a/src/schemas/src/dataset-schema.ts
+++ b/src/schemas/src/dataset-schema.ts
@@ -2,6 +2,7 @@
// Copyright contributors to the kepler.gl project
import pick from 'es-toolkit/compat/pick';
+import isPlainObject from 'es-toolkit/compat/isPlainObject';
import {console as globalConsole} from 'global/window';
import * as arrow from 'apache-arrow';
@@ -107,6 +108,47 @@ export const propertiesV1 = {
disableDataOperation: null
};
+/** Convert nested Arrow lists/structs and BigInts without changing live values. */
+function getValueForSaving(value: any, ancestors = new Set<object>()): any {
+ if (typeof value === 'bigint') {
+ const number = Number(value);
+ return Number.isSafeInteger(number) ? number : value.toString();
+ }
+ // Leave custom JSON representations intact; Arrow wrappers need normalization below.
+ if (
+ typeof value?.toJSON === 'function' &&
+ !(value instanceof arrow.StructRow) &&
+ !(value instanceof arrow.Vector)
+ ) {
+ return value;
+ }
+ const plain = value instanceof arrow.StructRow ? value.toJSON() : value;
+ const isList = Array.isArray(plain) || plain instanceof arrow.Vector;
+ if (!isList && !isPlainObject(plain)) return value;
+ if (ancestors.has(value)) throw new TypeError('Cannot export circular dataset values');
+ ancestors.add(value);
+ let saved = plain;
+ if (isList) {
+ const items = plain instanceof arrow.Vector ? Array.from(plain) : plain;
+ saved = items;
+ for (let i = 0; i < items.length; i++) {
+ const next = getValueForSaving(items[i], ancestors);
+ if (next === items[i]) continue;
+ if (saved === items) saved = items.slice();
+ saved[i] = next;
+ }
+ } else {
+ for (const key of Object.keys(plain)) {
+ const next = getValueForSaving(plain[key], ancestors);
+ if (next === plain[key]) continue;
+ if (saved === plain) saved = {...plain};
+ saved[key] = next;
+ }
+ }
+ ancestors.delete(value);
+ return saved;
+}
+
/**
* TODO Consider moving this cast to ArrowDataContainer?
* Prepare a data container for export as part of json / html files.
@@ -135,14 +177,6 @@ const getAllDataForSaving = (dataContainer: DataContainerInterface): any[][] =>
const value = row[columnIndex];
row[columnIndex] = value == null ? null : new Date(value).toISOString();
});
- } else if (arrow.DataType.isInt(column.type) && column.type.bitWidth === 64) {
- allData.forEach(row => {
- const value = row[columnIndex];
- if (typeof value === 'bigint') {
- const number = Number(value);
- row[columnIndex] = Number.isSafeInteger(number) ? number : value.toString();
- }
- });
} else if (field?.type === ALL_FIELD_TYPES.geoarrow) {
const formatter = FIELD_DISPLAY_FORMAT[ALL_FIELD_TYPES.geoarrow];
allData.forEach(row => {
@@ -150,27 +184,10 @@ const getAllDataForSaving = (dataContainer: DataContainerInterface): any[][] =>
});
}
}
- } else {
- // Row containers expose their live backing rows. Copy only when conversion
- // is needed, leaving other rows and exact stored values intact.
- let savedData = allData;
- for (let rowIndex = 0; rowIndex < allData.length; rowIndex++) {
- const row = allData[rowIndex];
- if (row == null) continue;
- let savedRow = row;
- for (let columnIndex = 0; columnIndex < row.length; columnIndex++) {
- const value = row[columnIndex];
- if (typeof value !== 'bigint') continue;
- if (savedData === allData) savedData = allData.slice();
- if (savedRow === row) savedRow = savedData[rowIndex] = row.slice();
- const number = Number(value);
- savedRow[columnIndex] = Number.isSafeInteger(number) ? number : value.toString();
- }
- }
- return savedData;
}
- return allData;
+ // Row containers expose live rows: copy only values that need conversion.
+ return getValueForSaving(allData);
};
/**
diff --git a/src/utils/src/export-utils.spec.ts b/src/utils/src/export-utils.spec.ts
index dc1e099d3..b1089c5f7 100644
--- a/src/utils/src/export-utils.spec.ts
+++ b/src/utils/src/export-utils.spec.ts
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import {exportJson} from './export-utils';
+import {exportJson, exportToJsonString} from './export-utils';
function readJsonBlob(blob: Blob): Promise<unknown> {
return new Promise((resolve, reject) => {
@@ -76,3 +76,28 @@ describe('exportJson', () => {
});
});
});
+
+describe('JSON export failures', () => {
+ test('propagates serialization failures and never downloads an error-only file', () => {
+ const cycle: any = {};
+ cycle.self = cycle;
+ const saveBlob = jest.fn();
+ Object.defineProperty(window.navigator, 'msSaveOrOpenBlob', {
+ configurable: true,
+ value: saveBlob
+ });
+ expect(() => exportToJsonString(cycle)).toThrow(TypeError);
+ expect(() => exportJson({visState: {schema: {save: () => cycle}}}, {hasData: true})).toThrow(
+ TypeError
+ );
+ expect(saveBlob).not.toHaveBeenCalled();
+ const failure = new Error('Cannot read export data');
+ expect(() =>
+ exportToJsonString({
+ toJSON() {
+ throw failure;
+ }
+ })
+ ).toThrow(failure);
+ });
+});
diff --git a/src/utils/src/export-utils.ts b/src/utils/src/export-utils.ts
index e488f8233..a26c52b39 100644
--- a/src/utils/src/export-utils.ts
+++ b/src/utils/src/export-utils.ts
@@ -167,13 +167,7 @@ export function exportImage(uiStateExportImage: ExportImage, filename?: string)
}
export function exportToJsonString(data) {
- try {
- return JSON.stringify(data);
- } catch (e) {
- if (e instanceof TypeError) return e.message;
- // Non-Standard Error Object Property
- return (e as any).description;
- }
+ return JSON.stringify(data);
}
/** Dataset metadata fields that hold a private layer API key. */
diff --git a/test/node/schemas/dataset-schema-test.js b/test/node/schemas/dataset-schema-test.js
index bf8c29ab4..ee95f63e9 100644
--- a/test/node/schemas/dataset-schema-test.js
+++ b/test/node/schemas/dataset-schema-test.js
@@ -345,6 +345,162 @@ test('#DatasetSchema -> Int64 values survive JSON, HTML and reload without preci
t.end();
});
+test('#DatasetSchema -> nested Arrow Int64 values survive a complete JSON backup and reload', t => {
+ const item = new arrow.Struct([
+ new arrow.Field('signed', new arrow.Int64(), true),
+ new arrow.Field('unsigned', new arrow.Uint64(), true)
+ ]);
+ const type = new arrow.Struct([
+ new arrow.Field('items', new arrow.List(new arrow.Field('item', item, true)), true)
+ ]);
+ const values = [
+ {
+ items: [
+ {signed: 42n, unsigned: 9007199254740991n},
+ {signed: -9223372036854775808n, unsigned: 18446744073709551615n},
+ {signed: null, unsigned: 0n},
+ null
+ ]
+ },
+ {items: []},
+ {items: null},
+ null
+ ];
+ const source = new arrow.Table({nested: arrow.vectorFromArray(values, type)});
+ const ipc = arrow.tableFromIPC(arrow.tableToIPC(source, 'file'));
+ const dataset = new KeplerTable({info: {id: 'nested'}, color: [1, 2, 3]}).updateSchema(
+ processArrowTable({shape: 'arrow-table', data: ipc})
+ );
+ const expected = [
+ [
+ {
+ items: [
+ {signed: 42, unsigned: 9007199254740991},
+ {signed: '-9223372036854775808', unsigned: '18446744073709551615'},
+ {signed: null, unsigned: 0},
+ null
+ ]
+ }
+ ],
+ [{items: []}],
+ [{items: null}],
+ [null]
+ ];
+ const saved = datasetSchema[VERSIONS.v1].save(dataset);
+ t.doesNotThrow(() => {
+ const payload = JSON.parse(
+ exportToJsonString({
+ datasets: [{version: 'v1', data: saved}],
+ config: {version: 'v1', config: {}}
+ })
+ );
+ t.deepEqual(payload.datasets[0].data.allData, expected, 'full backup has exact nested values');
+ const reloaded = reloadDataset(payload.datasets[0].data);
+ t.deepEqual(reloaded.dataContainer.flattenData(), expected, 'reload retains nesting and nulls');
+ t.deepEqual(datasetSchema[VERSIONS.v1].save(reloaded).allData, expected, 'resave is stable');
+ const html = exportMapToHTML(payload);
+ t.deepEqual(
+ JSON.parse(html.match(/const datasets = (.*);/)[1]),
+ payload.datasets,
+ 'HTML uses the same saved values'
+ );
+ }, 'backup is valid JSON and reloads');
+ t.equal(
+ dataset.dataContainer.valueAt(0, 0).items.get(1).signed,
+ -9223372036854775808n,
+ 'live Arrow values are not mutated'
+ );
+ t.equal(
+ ipc.getChild('nested').get(0).items.get(1).unsigned,
+ 18446744073709551615n,
+ 'source buffers are not mutated'
+ );
+
+ const rows = [[{items: [1n, null, {value: 9007199254740993n}]}]];
+ const rowTable = new KeplerTable({info: {id: 'nested-rows'}, color: [1, 2, 3]}).updateSchema({
+ fields: [{name: 'nested', type: 'object'}],
+ rows
+ });
+ t.deepEqual(
+ datasetSchema[VERSIONS.v1].save(rowTable).allData,
+ [[{items: [1, null, {value: '9007199254740993'}]}]],
+ 'row-backed nested values use the same policy'
+ );
+ t.equal(rows[0][0].items[2].value, 9007199254740993n, 'live row-backed objects are not mutated');
+ t.end();
+});
+
+test('#DatasetSchema -> nested custom JSON serializers retain their representations', t => {
+ class Identifier {
+ constructor() {
+ this.raw = 9007199254740993n;
+ }
+ toJSON() {
+ return {id: this.raw.toString()};
+ }
+ }
+ class CachedValue {
+ constructor() {
+ this.cache = {owner: this};
+ }
+ toJSON() {
+ return {value: 'valid'};
+ }
+ }
+ const custom = {
+ raw: 42n,
+ toJSON() {
+ return {value: this.raw.toString()};
+ }
+ };
+ custom.cache = custom;
+ const identifier = new Identifier();
+ const cached = new CachedValue();
+ const rows = [[{identifier, cached, custom, ordinary: {value: 42n}}]];
+ const dataset = new KeplerTable({info: {id: 'custom-json'}, color: [1, 2, 3]}).updateSchema({
+ fields: [{name: 'nested', type: 'object'}],
+ rows
+ });
+
+ const saved = datasetSchema[VERSIONS.v1].save(dataset);
+ const expected = [
+ [
+ {
+ identifier: {id: '9007199254740993'},
+ cached: {value: 'valid'},
+ custom: {value: '42'},
+ ordinary: {value: 42}
+ }
+ ]
+ ];
+ t.deepEqual(
+ JSON.parse(exportToJsonString(saved)).allData,
+ expected,
+ 'preserves class and plain-object serializers while normalizing ordinary nested values'
+ );
+ t.deepEqual(reloadDataset(saved).dataContainer.flattenData(), expected, 'custom values reload');
+ t.equal(saved.allData[0][0].identifier, identifier, 'retains the custom class instance');
+ t.equal(saved.allData[0][0].cached, cached, 'does not traverse the cyclic internal cache');
+ t.equal(identifier.raw, 9007199254740993n, 'does not change live class data');
+ t.equal(rows[0][0].ordinary.value, 42n, 'does not change ordinary live nested data');
+ t.end();
+});
+
+test('#DatasetSchema -> rejects ordinary circular values without a JSON serializer', t => {
+ const cycle = {};
+ cycle.self = cycle;
+ const dataset = new KeplerTable({info: {id: 'circular'}, color: [1, 2, 3]}).updateSchema({
+ fields: [{name: 'nested', type: 'object'}],
+ rows: [[cycle]]
+ });
+ t.throws(
+ () => datasetSchema[VERSIONS.v1].save(dataset),
+ /Cannot export circular dataset values/,
+ 'still rejects an actual serialization cycle'
+ );
+ t.end();
+});
+
test('#DatasetSchema -> joined Int64 rows save and reload without changing live rows', t => {
const values = [
0n,