Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[Bug] Preserve Arrow column order through reconstruction (#3889)
Igor Dykhtacba3fa06c485
diff --git a/src/duckdb/src/table/duckdb-table-utils.ts b/src/duckdb/src/table/duckdb-table-utils.tsindex f51af590f..694eaf188 100644--- a/src/duckdb/src/table/duckdb-table-utils.ts+++ b/src/duckdb/src/table/duckdb-table-utils.ts@@ -185,7 +185,13 @@ export const restoreArrowTable = (creaOpts[field.name] = columns[index];});- return arrowSchema ? new arrow.Table(arrowSchema, creaOpts) : new arrow.Table(creaOpts);+ const ordered = new arrow.Table(creaOpts).select(fields.map(field => field.name));+ return arrowSchema+ ? new arrow.Table(+ arrowSchema,+ ordered.batches.map(batch => new arrow.RecordBatch(arrowSchema, batch.data))+ )+ : ordered;};/**diff --git a/src/reducers/src/column-edit-updaters.spec.ts b/src/reducers/src/column-edit-updaters.spec.tsindex 01292894e..8df0ae3f5 100644--- a/src/reducers/src/column-edit-updaters.spec.ts+++ b/src/reducers/src/column-edit-updaters.spec.ts@@ -11,6 +11,14 @@ import {import {VisStateActions} from '@kepler.gl/actions';import {KeplerTable} from '@kepler.gl/table';import SchemaManager, {VisState} from '@kepler.gl/schemas';+import {processArrowBatches} from '@kepler.gl/processors';+import {+ exportMapToHTML,+ exportToJsonString,+ getMapJSON,+ ArrowDataContainer+} from '@kepler.gl/utils';+import * as arrow from 'apache-arrow';import {drainTasksForTesting, errorTaskInTest, succeedTaskInTest} from '@kepler.gl/tasks';import type {PolygonFilter, RangeFilter, TimeRangeFilter} from '@kepler.gl/types';@@ -19,6 +27,101 @@ import {INITIAL_VIS_STATE} from './vis-state-updaters';const reduceVisState = visStateReducer as (state: VisState, action: unknown) => VisState;+function arrowCities() {+ const source = new arrow.Table({+ lat: arrow.vectorFromArray([37, null], new arrow.Float64()),+ lng: arrow.vectorFromArray([-122, -123], new arrow.Float64()),+ population: arrow.vectorFromArray([1000, null], new arrow.Int32())+ });+ const data = processArrowBatches(source.batches);+ if (!data) throw new Error('Expected Arrow data');+ return reduceVisState(+ withTables(makeTable('cities', data)),+ VisStateActions.addLayer({+ id: 'cities-layer',+ type: 'point',+ config: {+ dataId: 'cities',+ columns: {lat: 'lat', lng: 'lng'},+ colorField: {name: 'population', type: ALL_FIELD_TYPES.integer}+ }+ })+ );+}++describe('Arrow column order through edits and saved maps', () => {+ test.each(['0', '2020', '4294967294', '00', '02020', '-1', '1.0', '1e3', '4294967295'])(+ 'rename to %s preserves descriptor/vector order',+ name => {+ const initial = arrowCities();+ const original = initial.datasets.cities.dataContainer as ArrowDataContainer;+ const next = reduceVisState(+ initial,+ VisStateActions.renameTableColumn('cities', 'population', name)+ );+ const table = next.datasets.cities;+ const dc = table.dataContainer as ArrowDataContainer;+ expect(table.fields.map(f => f.name)).toEqual(['lat', 'lng', name]);+ expect(dc.getTable().schema.fields.map(f => f.name)).toEqual(['lat', 'lng', name]);+ expect(dc.flattenData()).toEqual([+ [37, -122, 1000],+ [null, -123, null]+ ]);+ expect(table.fields.map(f => f.valueAccessor({index: 0}))).toEqual([37, -122, 1000]);+ table.fields.forEach((_, i) => {+ expect(dc.getColumn(i).data[0].values.buffer).toBe(+ original.getColumn(i).data[0].values.buffer+ );+ });+ expect(initial.datasets.cities.fields[2].name).toBe('population');+ expect(original.getTable().schema.fields[2].name).toBe('population');+ expect(original.rowAsArray(0)).toEqual([37, -122, 1000]);+ const layer = next.layers[0];+ expect(layer.getPositionAccessor(dc)({index: 0})).toEqual([-122, 37, 0]);+ layer.formatLayerData(next.datasets);+ expect(Array.from((layer as any).geoArrowVector.get(0))).toEqual([-122, 37, 0]);+ expect(layer.config.colorField?.name).toBe(name);+ }+ );++ test('JSON, HTML and native IPC reload retain the renamed values and positions', () => {+ const next = reduceVisState(+ arrowCities(),+ VisStateActions.renameTableColumn('cities', 'population', '2020')+ );+ const saved = getMapJSON(+ {visState: next, mapState: {}, mapStyle: {}, uiState: {}},+ {hasData: true}+ );+ const json = JSON.parse(exportToJsonString(saved));+ const html = exportMapToHTML({...saved, mode: 'READ'});+ const datasetsMatch = html.match(/const datasets = (.+);/);+ const configMatch = html.match(/const config = (.+);/);+ if (!datasetsMatch || !configMatch) throw new Error('Expected embedded map data');+ const htmlData = JSON.parse(datasetsMatch[1]);+ const htmlConfig = JSON.parse(configMatch[1]);+ for (const document of [json, {datasets: htmlData, config: htmlConfig}]) {+ const loaded = SchemaManager.load(document);+ const dataset = loaded.datasets?.[0];+ if (!dataset) throw new Error('Expected reloaded dataset');+ const table = makeTable('cities', dataset.data);+ expect(table.fields.map(f => f.name)).toEqual(['lat', 'lng', '2020']);+ expect(table.dataContainer.rowAsArray(0)).toEqual([37, -122, 1000]);+ expect(next.layers[0].getPositionAccessor(table.dataContainer)({index: 0})).toEqual([+ -122, 37, 0+ ]);+ }+ const dc = next.datasets.cities.dataContainer as ArrowDataContainer;+ const ipc = arrow.tableFromIPC(arrow.tableToIPC(dc.getTable()));+ const processed = processArrowBatches(ipc.batches);+ if (!processed) throw new Error('Expected IPC data');+ const restored = makeTable('cities', processed);+ expect(restored.dataContainer.rowAsArray(0)).toEqual([37, -122, 1000]);+ expect(ipc.schema.fields.map(f => f.name)).toEqual(['lat', 'lng', '2020']);+ expect(ipc.getChild('2020')?.get(0)).toBe(1000);+ });+});+function makeTable(id: string, data: Parameters<KeplerTable['updateSchema']>[0]) {const table = new KeplerTable({info: {id, label: id, type: 'local'},diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.tsindex a4cfc9712..46ae8e6f3 100644--- a/src/utils/src/arrow-data-container.ts+++ b/src/utils/src/arrow-data-container.ts@@ -165,6 +165,7 @@ export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number):try {const columns: Record<string, arrow.Vector> = {};+ const columnNames: string[] = [];const seenNames = new Map<string, number>();for (let i = 0; i < table.numCols; i++) {const field = table.schema.fields[i];@@ -177,12 +178,13 @@ export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number):seenNames.set(field.name, seen + 1);const key = seen === 0 ? field.name : `${field.name}_${seen}`;columns[key] = compactArrowVector(column, column.type || field.type);+ columnNames.push(key);}// Always build a kepler apache-arrow Table. Using `table.constructor` can// mix DuckDB's bundled Arrow Table with rebuilt vectors and drop schema// metadata (`geo`, ARROW:extension:name) that Kepler needs to create layers.- const compacted = tableFromCompactedColumns(table, columns);+ const compacted = tableFromCompactedColumns(table, columns, columnNames);if (compacted?.batches?.length && compacted.batches.length < table.batches.length) {return compacted;@@ -228,13 +230,18 @@ function copyArrowSchemaMetadata(from?: arrow.Schema | null, to?: arrow.Schema |function tableFromCompactedColumns(source: arrow.Table,- columns: Record<string, arrow.Vector>+ columns: Record<string, arrow.Vector>,+ columnNames: string[]): arrow.Table {+ const ordered = new arrow.Table(columns).select(columnNames);let compacted: arrow.Table;try {- compacted = new arrow.Table(source.schema, columns);+ compacted = new arrow.Table(+ source.schema,+ ordered.batches.map(batch => new arrow.RecordBatch(source.schema, batch.data))+ );} catch {- compacted = new arrow.Table(columns);+ compacted = ordered;}copyArrowSchemaMetadata(source.schema, compacted.schema);return compacted;@@ -366,7 +373,7 @@ export class ArrowDataContainer implements DataContainerInterface {this._fields.map((field, index) => {creaOpts[field.name] = this._cols[index];});- return new arrow.Table(creaOpts);+ return new arrow.Table(creaOpts).select(this._fields.map(field => field.name));}getTable() {diff --git a/test/node/utils/data-container-test.js b/test/node/utils/data-container-test.jsindex 8d99b3f7b..14ac06bb2 100644--- a/test/node/utils/data-container-test.js+++ b/test/node/utils/data-container-test.js@@ -12,6 +12,88 @@ import {import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants';import * as arrow from 'apache-arrow';+function orderedNumericTable() {+ const cols = [+ arrow.vectorFromArray([37, null, 38], new arrow.Float64()),+ arrow.vectorFromArray([-122, -123, -124], new arrow.Float32()),+ arrow.vectorFromArray([1000, null, 2000], new arrow.Int32())+ ];+ const fields = cols.map(+ (col, i) =>+ new arrow.Field(['lat', 'lng', '2020'][i], col.type, true, new Map([['source', String(i)]]))+ );+ const schema = new arrow.Schema(fields, new Map([['source', 'ordered']]));+ const batch = new arrow.RecordBatch(+ schema,+ arrow.makeData({type: new arrow.Struct(fields), length: 3, children: cols.map(c => c.data[0])})+ );+ return new arrow.Table(schema, [batch]);+}++test('ArrowDataContainer -> numeric headers preserve vector order through updates and IPC', t => {+ const source = orderedNumericTable();+ const cols = source.schema.fields.map((_, i) => source.getChildAt(i));+ const dc = new ArrowDataContainer({cols, fields: source.schema.fields});+ const rows = [+ [37, -122, 1000],+ [null, -123, null],+ [38, -124, 2000]+ ];+ t.deepEqual(dc.flattenData(), rows, 'numeric names do not reorder values');+ dc.update([+ cols[0].slice(0, 1).concat(cols[0].slice(1)),+ cols[1],+ cols[2].slice(0, 2).concat(cols[2].slice(2))+ ]);+ t.equal(dc.numChunks(), 3, 'staggered chunks remain aligned');+ t.deepEqual(dc.flattenData(), rows);+ t.deepEqual(+ dc.getTable().schema.fields.map(f => f.name),+ ['lat', 'lng', '2020']+ );+ cols.forEach((col, i) => {+ t.equal(dc.getColumn(i).type.toString(), col.type.toString(), 'physical types stay aligned');+ dc.getColumn(i).data.forEach(chunk => {+ t.equal(chunk.values.buffer, col.data[0].values.buffer, 'values are shared');+ t.equal(chunk.nullBitmap.buffer, col.data[0].nullBitmap.buffer, 'validity is shared');+ });+ });+ const restored = arrow.tableFromIPC(arrow.tableToIPC(dc.getTable()));+ t.equal(restored.getChild('lat').get(0), 37);+ t.equal(restored.getChild('2020').get(0), 1000);+ dc.update(cols.map(col => col.slice(0, 0)));+ t.equal(dc.numRows(), 0);+ t.equal(dc.numColumns(), 3);+ t.deepEqual(+ dc.getTable().schema.fields.map(f => f.name),+ ['lat', 'lng', '2020']+ );+ t.equal(dc.getColumn(2).type.toString(), 'Int32', 'empty columns keep their types');+ t.end();+});++test('compactArrowTable -> numeric headers keep values aligned with the schema', t => {+ const source = orderedNumericTable();+ const compacted = compactArrowTable(source.concat(source), 1);+ t.equal(compacted.batches.length, 1);+ t.deepEqual(+ compacted.schema.fields.map(f => f.name),+ ['lat', 'lng', '2020']+ );+ source.schema.fields.forEach((field, i) => {+ t.deepEqual(Array.from(compacted.getChildAt(i)), [+ ...source.getChildAt(i),+ ...source.getChildAt(i)+ ]);+ t.equal(compacted.getChildAt(i).type.toString(), field.type.toString());+ t.deepEqual(compacted.schema.fields[i].metadata, field.metadata);+ });+ t.deepEqual(compacted.schema.metadata, source.schema.metadata);+ t.equal(arrow.tableFromIPC(arrow.tableToIPC(compacted)).getChild('2020').get(0), 1000);+ t.equal(source.getChild('lat').get(0), 37, 'source values remain untouched');+ t.end();+});+const data = [[10, 20], // 0[30, 40], // 1diff --git a/test/node/utils/duckdb-utils-test.js b/test/node/utils/duckdb-utils-test.jsindex 9924bc226..c59e29147 100644--- a/test/node/utils/duckdb-utils-test.js+++ b/test/node/utils/duckdb-utils-test.js@@ -10,6 +10,7 @@ import {castDuckDBTypesForKepler,setGeoArrowWKBExtension,removeUnsupportedExtensions,+ restoreArrowTable,restoreUnsupportedExtensions,isGeoArrowPoint,isGeoArrowLineString,@@ -25,6 +26,43 @@ import {import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants';+test('restoreArrowTable -> numeric names preserve column order and supplied schema', t => {+ const columns = [+ arrow.vectorFromArray([37, null], new arrow.Float64()),+ arrow.vectorFromArray([1000, 2000], new arrow.Int32())+ ];+ const fields = ['lat', '2020'].map(+ (name, i) => new arrow.Field(name, columns[i].type, true, new Map([['source', name]]))+ );+ const schema = new arrow.Schema(fields, new Map([['source', 'original']]));+ for (const originalSchema of [undefined, schema]) {+ const restored = restoreArrowTable(columns, fields, originalSchema);+ t.deepEqual(+ restored.schema.fields.map(f => f.name),+ ['lat', '2020']+ );+ t.deepEqual(Array.from(restored.getChildAt(0)), [37, null]);+ t.deepEqual(Array.from(restored.getChildAt(1)), [1000, 2000]);+ columns.forEach((column, i) => {+ t.equal(restored.getChildAt(i).type.toString(), column.type.toString());+ t.equal(restored.getChildAt(i).data[0].values.buffer, column.data[0].values.buffer);+ });+ if (originalSchema) {+ t.deepEqual(restored.schema.metadata, schema.metadata);+ t.deepEqual(restored.schema.fields[0].metadata, fields[0].metadata);+ }+ t.equal(arrow.tableFromIPC(arrow.tableToIPC(restored)).getChild('2020').get(0), 1000);+ const empty = restoreArrowTable(+ columns.map(column => column.slice(0, 0)),+ fields,+ originalSchema+ );+ t.equal(empty.numRows, 0);+ t.equal(empty.getChildAt(1).type.toString(), 'Int32');+ }+ t.end();+});+// Test getDuckDBColumnTypesMaptest('duckdb-utils -> getDuckDBColumnTypesMap', t => {const columns = [