Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[Bug] Preserve Arrow column order through reconstruction (#3889)

Igor Dykhtacba3fa06c485
diff --git a/src/duckdb/src/table/duckdb-table-utils.ts b/src/duckdb/src/table/duckdb-table-utils.ts
index f51af590f..694eaf188 100644
--- a/src/duckdb/src/table/duckdb-table-utils.ts
+++ b/src/duckdb/src/table/duckdb-table-utils.ts
@@ -185,7 +185,13 @@ export const restoreArrowTable = (
creaOpts[field.name] = columns[index];
});
- return arrowSchema ? new arrow.Table(arrowSchema, creaOpts) : new arrow.Table(creaOpts);
+ const ordered = new arrow.Table(creaOpts).select(fields.map(field => field.name));
+ return arrowSchema
+ ? new arrow.Table(
+ arrowSchema,
+ ordered.batches.map(batch => new arrow.RecordBatch(arrowSchema, batch.data))
+ )
+ : ordered;
};
/**
diff --git a/src/reducers/src/column-edit-updaters.spec.ts b/src/reducers/src/column-edit-updaters.spec.ts
index 01292894e..8df0ae3f5 100644
--- a/src/reducers/src/column-edit-updaters.spec.ts
+++ b/src/reducers/src/column-edit-updaters.spec.ts
@@ -11,6 +11,14 @@ import {
import {VisStateActions} from '@kepler.gl/actions';
import {KeplerTable} from '@kepler.gl/table';
import SchemaManager, {VisState} from '@kepler.gl/schemas';
+import {processArrowBatches} from '@kepler.gl/processors';
+import {
+ exportMapToHTML,
+ exportToJsonString,
+ getMapJSON,
+ ArrowDataContainer
+} from '@kepler.gl/utils';
+import * as arrow from 'apache-arrow';
import {drainTasksForTesting, errorTaskInTest, succeedTaskInTest} from '@kepler.gl/tasks';
import type {PolygonFilter, RangeFilter, TimeRangeFilter} from '@kepler.gl/types';
@@ -19,6 +27,101 @@ import {INITIAL_VIS_STATE} from './vis-state-updaters';
const reduceVisState = visStateReducer as (state: VisState, action: unknown) => VisState;
+function arrowCities() {
+ const source = new arrow.Table({
+ lat: arrow.vectorFromArray([37, null], new arrow.Float64()),
+ lng: arrow.vectorFromArray([-122, -123], new arrow.Float64()),
+ population: arrow.vectorFromArray([1000, null], new arrow.Int32())
+ });
+ const data = processArrowBatches(source.batches);
+ if (!data) throw new Error('Expected Arrow data');
+ return reduceVisState(
+ withTables(makeTable('cities', data)),
+ VisStateActions.addLayer({
+ id: 'cities-layer',
+ type: 'point',
+ config: {
+ dataId: 'cities',
+ columns: {lat: 'lat', lng: 'lng'},
+ colorField: {name: 'population', type: ALL_FIELD_TYPES.integer}
+ }
+ })
+ );
+}
+
+describe('Arrow column order through edits and saved maps', () => {
+ test.each(['0', '2020', '4294967294', '00', '02020', '-1', '1.0', '1e3', '4294967295'])(
+ 'rename to %s preserves descriptor/vector order',
+ name => {
+ const initial = arrowCities();
+ const original = initial.datasets.cities.dataContainer as ArrowDataContainer;
+ const next = reduceVisState(
+ initial,
+ VisStateActions.renameTableColumn('cities', 'population', name)
+ );
+ const table = next.datasets.cities;
+ const dc = table.dataContainer as ArrowDataContainer;
+ expect(table.fields.map(f => f.name)).toEqual(['lat', 'lng', name]);
+ expect(dc.getTable().schema.fields.map(f => f.name)).toEqual(['lat', 'lng', name]);
+ expect(dc.flattenData()).toEqual([
+ [37, -122, 1000],
+ [null, -123, null]
+ ]);
+ expect(table.fields.map(f => f.valueAccessor({index: 0}))).toEqual([37, -122, 1000]);
+ table.fields.forEach((_, i) => {
+ expect(dc.getColumn(i).data[0].values.buffer).toBe(
+ original.getColumn(i).data[0].values.buffer
+ );
+ });
+ expect(initial.datasets.cities.fields[2].name).toBe('population');
+ expect(original.getTable().schema.fields[2].name).toBe('population');
+ expect(original.rowAsArray(0)).toEqual([37, -122, 1000]);
+ const layer = next.layers[0];
+ expect(layer.getPositionAccessor(dc)({index: 0})).toEqual([-122, 37, 0]);
+ layer.formatLayerData(next.datasets);
+ expect(Array.from((layer as any).geoArrowVector.get(0))).toEqual([-122, 37, 0]);
+ expect(layer.config.colorField?.name).toBe(name);
+ }
+ );
+
+ test('JSON, HTML and native IPC reload retain the renamed values and positions', () => {
+ const next = reduceVisState(
+ arrowCities(),
+ VisStateActions.renameTableColumn('cities', 'population', '2020')
+ );
+ const saved = getMapJSON(
+ {visState: next, mapState: {}, mapStyle: {}, uiState: {}},
+ {hasData: true}
+ );
+ const json = JSON.parse(exportToJsonString(saved));
+ const html = exportMapToHTML({...saved, mode: 'READ'});
+ const datasetsMatch = html.match(/const datasets = (.+);/);
+ const configMatch = html.match(/const config = (.+);/);
+ if (!datasetsMatch || !configMatch) throw new Error('Expected embedded map data');
+ const htmlData = JSON.parse(datasetsMatch[1]);
+ const htmlConfig = JSON.parse(configMatch[1]);
+ for (const document of [json, {datasets: htmlData, config: htmlConfig}]) {
+ const loaded = SchemaManager.load(document);
+ const dataset = loaded.datasets?.[0];
+ if (!dataset) throw new Error('Expected reloaded dataset');
+ const table = makeTable('cities', dataset.data);
+ expect(table.fields.map(f => f.name)).toEqual(['lat', 'lng', '2020']);
+ expect(table.dataContainer.rowAsArray(0)).toEqual([37, -122, 1000]);
+ expect(next.layers[0].getPositionAccessor(table.dataContainer)({index: 0})).toEqual([
+ -122, 37, 0
+ ]);
+ }
+ const dc = next.datasets.cities.dataContainer as ArrowDataContainer;
+ const ipc = arrow.tableFromIPC(arrow.tableToIPC(dc.getTable()));
+ const processed = processArrowBatches(ipc.batches);
+ if (!processed) throw new Error('Expected IPC data');
+ const restored = makeTable('cities', processed);
+ expect(restored.dataContainer.rowAsArray(0)).toEqual([37, -122, 1000]);
+ expect(ipc.schema.fields.map(f => f.name)).toEqual(['lat', 'lng', '2020']);
+ expect(ipc.getChild('2020')?.get(0)).toBe(1000);
+ });
+});
+
function makeTable(id: string, data: Parameters<KeplerTable['updateSchema']>[0]) {
const table = new KeplerTable({
info: {id, label: id, type: 'local'},
diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.ts
index a4cfc9712..46ae8e6f3 100644
--- a/src/utils/src/arrow-data-container.ts
+++ b/src/utils/src/arrow-data-container.ts
@@ -165,6 +165,7 @@ export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number):
try {
const columns: Record<string, arrow.Vector> = {};
+ const columnNames: string[] = [];
const seenNames = new Map<string, number>();
for (let i = 0; i < table.numCols; i++) {
const field = table.schema.fields[i];
@@ -177,12 +178,13 @@ export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number):
seenNames.set(field.name, seen + 1);
const key = seen === 0 ? field.name : `${field.name}_${seen}`;
columns[key] = compactArrowVector(column, column.type || field.type);
+ columnNames.push(key);
}
// Always build a kepler apache-arrow Table. Using `table.constructor` can
// mix DuckDB's bundled Arrow Table with rebuilt vectors and drop schema
// metadata (`geo`, ARROW:extension:name) that Kepler needs to create layers.
- const compacted = tableFromCompactedColumns(table, columns);
+ const compacted = tableFromCompactedColumns(table, columns, columnNames);
if (compacted?.batches?.length && compacted.batches.length < table.batches.length) {
return compacted;
@@ -228,13 +230,18 @@ function copyArrowSchemaMetadata(from?: arrow.Schema | null, to?: arrow.Schema |
function tableFromCompactedColumns(
source: arrow.Table,
- columns: Record<string, arrow.Vector>
+ columns: Record<string, arrow.Vector>,
+ columnNames: string[]
): arrow.Table {
+ const ordered = new arrow.Table(columns).select(columnNames);
let compacted: arrow.Table;
try {
- compacted = new arrow.Table(source.schema, columns);
+ compacted = new arrow.Table(
+ source.schema,
+ ordered.batches.map(batch => new arrow.RecordBatch(source.schema, batch.data))
+ );
} catch {
- compacted = new arrow.Table(columns);
+ compacted = ordered;
}
copyArrowSchemaMetadata(source.schema, compacted.schema);
return compacted;
@@ -366,7 +373,7 @@ export class ArrowDataContainer implements DataContainerInterface {
this._fields.map((field, index) => {
creaOpts[field.name] = this._cols[index];
});
- return new arrow.Table(creaOpts);
+ return new arrow.Table(creaOpts).select(this._fields.map(field => field.name));
}
getTable() {
diff --git a/test/node/utils/data-container-test.js b/test/node/utils/data-container-test.js
index 8d99b3f7b..14ac06bb2 100644
--- a/test/node/utils/data-container-test.js
+++ b/test/node/utils/data-container-test.js
@@ -12,6 +12,88 @@ import {
import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants';
import * as arrow from 'apache-arrow';
+function orderedNumericTable() {
+ const cols = [
+ arrow.vectorFromArray([37, null, 38], new arrow.Float64()),
+ arrow.vectorFromArray([-122, -123, -124], new arrow.Float32()),
+ arrow.vectorFromArray([1000, null, 2000], new arrow.Int32())
+ ];
+ const fields = cols.map(
+ (col, i) =>
+ new arrow.Field(['lat', 'lng', '2020'][i], col.type, true, new Map([['source', String(i)]]))
+ );
+ const schema = new arrow.Schema(fields, new Map([['source', 'ordered']]));
+ const batch = new arrow.RecordBatch(
+ schema,
+ arrow.makeData({type: new arrow.Struct(fields), length: 3, children: cols.map(c => c.data[0])})
+ );
+ return new arrow.Table(schema, [batch]);
+}
+
+test('ArrowDataContainer -> numeric headers preserve vector order through updates and IPC', t => {
+ const source = orderedNumericTable();
+ const cols = source.schema.fields.map((_, i) => source.getChildAt(i));
+ const dc = new ArrowDataContainer({cols, fields: source.schema.fields});
+ const rows = [
+ [37, -122, 1000],
+ [null, -123, null],
+ [38, -124, 2000]
+ ];
+ t.deepEqual(dc.flattenData(), rows, 'numeric names do not reorder values');
+ dc.update([
+ cols[0].slice(0, 1).concat(cols[0].slice(1)),
+ cols[1],
+ cols[2].slice(0, 2).concat(cols[2].slice(2))
+ ]);
+ t.equal(dc.numChunks(), 3, 'staggered chunks remain aligned');
+ t.deepEqual(dc.flattenData(), rows);
+ t.deepEqual(
+ dc.getTable().schema.fields.map(f => f.name),
+ ['lat', 'lng', '2020']
+ );
+ cols.forEach((col, i) => {
+ t.equal(dc.getColumn(i).type.toString(), col.type.toString(), 'physical types stay aligned');
+ dc.getColumn(i).data.forEach(chunk => {
+ t.equal(chunk.values.buffer, col.data[0].values.buffer, 'values are shared');
+ t.equal(chunk.nullBitmap.buffer, col.data[0].nullBitmap.buffer, 'validity is shared');
+ });
+ });
+ const restored = arrow.tableFromIPC(arrow.tableToIPC(dc.getTable()));
+ t.equal(restored.getChild('lat').get(0), 37);
+ t.equal(restored.getChild('2020').get(0), 1000);
+ dc.update(cols.map(col => col.slice(0, 0)));
+ t.equal(dc.numRows(), 0);
+ t.equal(dc.numColumns(), 3);
+ t.deepEqual(
+ dc.getTable().schema.fields.map(f => f.name),
+ ['lat', 'lng', '2020']
+ );
+ t.equal(dc.getColumn(2).type.toString(), 'Int32', 'empty columns keep their types');
+ t.end();
+});
+
+test('compactArrowTable -> numeric headers keep values aligned with the schema', t => {
+ const source = orderedNumericTable();
+ const compacted = compactArrowTable(source.concat(source), 1);
+ t.equal(compacted.batches.length, 1);
+ t.deepEqual(
+ compacted.schema.fields.map(f => f.name),
+ ['lat', 'lng', '2020']
+ );
+ source.schema.fields.forEach((field, i) => {
+ t.deepEqual(Array.from(compacted.getChildAt(i)), [
+ ...source.getChildAt(i),
+ ...source.getChildAt(i)
+ ]);
+ t.equal(compacted.getChildAt(i).type.toString(), field.type.toString());
+ t.deepEqual(compacted.schema.fields[i].metadata, field.metadata);
+ });
+ t.deepEqual(compacted.schema.metadata, source.schema.metadata);
+ t.equal(arrow.tableFromIPC(arrow.tableToIPC(compacted)).getChild('2020').get(0), 1000);
+ t.equal(source.getChild('lat').get(0), 37, 'source values remain untouched');
+ t.end();
+});
+
const data = [
[10, 20], // 0
[30, 40], // 1
diff --git a/test/node/utils/duckdb-utils-test.js b/test/node/utils/duckdb-utils-test.js
index 9924bc226..c59e29147 100644
--- a/test/node/utils/duckdb-utils-test.js
+++ b/test/node/utils/duckdb-utils-test.js
@@ -10,6 +10,7 @@ import {
castDuckDBTypesForKepler,
setGeoArrowWKBExtension,
removeUnsupportedExtensions,
+ restoreArrowTable,
restoreUnsupportedExtensions,
isGeoArrowPoint,
isGeoArrowLineString,
@@ -25,6 +26,43 @@ import {
import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants';
+test('restoreArrowTable -> numeric names preserve column order and supplied schema', t => {
+ const columns = [
+ arrow.vectorFromArray([37, null], new arrow.Float64()),
+ arrow.vectorFromArray([1000, 2000], new arrow.Int32())
+ ];
+ const fields = ['lat', '2020'].map(
+ (name, i) => new arrow.Field(name, columns[i].type, true, new Map([['source', name]]))
+ );
+ const schema = new arrow.Schema(fields, new Map([['source', 'original']]));
+ for (const originalSchema of [undefined, schema]) {
+ const restored = restoreArrowTable(columns, fields, originalSchema);
+ t.deepEqual(
+ restored.schema.fields.map(f => f.name),
+ ['lat', '2020']
+ );
+ t.deepEqual(Array.from(restored.getChildAt(0)), [37, null]);
+ t.deepEqual(Array.from(restored.getChildAt(1)), [1000, 2000]);
+ columns.forEach((column, i) => {
+ t.equal(restored.getChildAt(i).type.toString(), column.type.toString());
+ t.equal(restored.getChildAt(i).data[0].values.buffer, column.data[0].values.buffer);
+ });
+ if (originalSchema) {
+ t.deepEqual(restored.schema.metadata, schema.metadata);
+ t.deepEqual(restored.schema.fields[0].metadata, fields[0].metadata);
+ }
+ t.equal(arrow.tableFromIPC(arrow.tableToIPC(restored)).getChild('2020').get(0), 1000);
+ const empty = restoreArrowTable(
+ columns.map(column => column.slice(0, 0)),
+ fields,
+ originalSchema
+ );
+ t.equal(empty.numRows, 0);
+ t.equal(empty.getChildAt(1).type.toString(), 'Int32');
+ }
+ t.end();
+});
+
// Test getDuckDBColumnTypesMap
test('duckdb-utils -> getDuckDBColumnTypesMap', t => {
const columns = [