Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[Bug] fix Arrow Int64 Group By arithmetic (#3876)

Igor Dykhta12cc827424bf
diff --git a/src/table/src/dataset-ops.spec.ts b/src/table/src/dataset-ops.spec.ts
index dfebe31d7..fa6a4a756 100644
--- a/src/table/src/dataset-ops.spec.ts
+++ b/src/table/src/dataset-ops.spec.ts
@@ -3,8 +3,10 @@
import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
import {latLngToCell} from '@kepler.gl/common-utils';
+import {Int64, Table, Uint64, vectorFromArray} from 'apache-arrow';
import KeplerTable from './kepler-table';
+import {aggregateValues} from './dataset-ops/aggregations';
import {
DATASET_OPS_AGGREGATIONS,
aggregationOptionsForField,
@@ -46,6 +48,20 @@ function makeTable({
return table;
}
+function makeInt64Data(values: (bigint | null)[], type: Int64 | Uint64 = new Int64()) {
+ const region = vectorFromArray(values.map(value => (value === null ? 'missing' : 'west')));
+ const pop = vectorFromArray(values, type);
+ return {
+ fields: [
+ {name: 'region', type: ALL_FIELD_TYPES.string},
+ {name: 'pop', type: ALL_FIELD_TYPES.integer}
+ ],
+ cols: [region, pop],
+ rows: [],
+ arrowTable: new Table({region, pop})
+ };
+}
+
const square = (west: number, south: number, east: number, north: number) => ({
type: 'Feature',
properties: {},
@@ -101,6 +117,79 @@ describe('dataset-ops engine', () => {
expect(byRegion.east).toEqual(['east', 40, 1]);
});
+ describe.each([new Int64(), new Uint64()])('Arrow %s Group By', arrowType => {
+ test.each([
+ [DATASET_OPS_AGGREGATIONS.sum, 30, 80],
+ [DATASET_OPS_AGGREGATIONS.average, 15, 40],
+ [DATASET_OPS_AGGREGATIONS.median, 15, 40]
+ ] as const)('%s works before and after repeated updates', async (technique, first, next) => {
+ const dataset = new KeplerTable({info: {id: 'int64'}, color: [0, 0, 0]});
+ dataset.updateSchema(makeInt64Data([10n, 20n, null], arrowType));
+ const config = {fieldName: 'region', aggregations: {pop: technique}};
+
+ expect(groupByDataset(dataset, config).data.rows).toEqual([
+ ['west', first],
+ ['missing', null]
+ ]);
+
+ for (let i = 0; i < 2; i++) {
+ const data = makeInt64Data([30n, 50n, null], arrowType);
+ await dataset.update(data);
+ const result = groupByDataset(dataset, config);
+ expect(result.data.rows).toEqual([
+ ['west', next],
+ ['missing', null]
+ ]);
+ expect([0, 1, 2].map(index => dataset.dataContainer.valueAt(index, 1))).toEqual([
+ 30n,
+ 50n,
+ null
+ ]);
+ expect([0, 1, 2].map(index => data.cols[1].get(index))).toEqual([30n, 50n, null]);
+
+ const reloaded = new KeplerTable({info: result.info, color: [0, 0, 0]});
+ await reloaded.importData({data: result.data});
+ expect(reloaded.dataContainer.valueAt(0, 1)).toBe(next);
+ }
+ });
+ });
+
+ test.each([
+ [DATASET_OPS_AGGREGATIONS.sum, 18014398509481984],
+ [DATASET_OPS_AGGREGATIONS.average, 9007199254740992],
+ [DATASET_OPS_AGGREGATIONS.median, 9007199254740992],
+ [DATASET_OPS_AGGREGATIONS.minimum, 9007199254740992n],
+ [DATASET_OPS_AGGREGATIONS.maximum, 9007199254740993n],
+ [DATASET_OPS_AGGREGATIONS.countUnique, 2],
+ [DATASET_OPS_AGGREGATIONS.count, 2]
+ ] as const)('Int64 %s preserves source precision', (technique, expected) => {
+ const values = [9007199254740992n, 9007199254740993n];
+ const data = makeInt64Data(values);
+ const dataset = new KeplerTable({info: {id: 'int64'}, color: [0, 0, 0]});
+ dataset.updateSchema(data);
+
+ const result = groupByDataset(dataset, {
+ fieldName: 'region',
+ aggregations: {pop: technique}
+ });
+
+ // Arithmetic uses Number precision; extrema and distinct counts retain exact BigInts.
+ expect(result.data.rows).toEqual([['west', expected]]);
+ expect([0, 1].map(index => dataset.dataContainer.valueAt(index, 1))).toEqual(values);
+ expect([0, 1].map(index => data.cols[1].get(index))).toEqual(values);
+ });
+
+ test.each([
+ [DATASET_OPS_AGGREGATIONS.sum, 30, 1],
+ [DATASET_OPS_AGGREGATIONS.average, 15, 0.5],
+ [DATASET_OPS_AGGREGATIONS.median, 15, 0.5]
+ ] as const)('%s handles mixed numbers and missing values', (technique, numeric, boolean) => {
+ expect(aggregateValues([10n, null, 20, undefined], technique)).toBe(numeric);
+ expect(aggregateValues([true, false, null], technique)).toBe(boolean);
+ expect(aggregateValues([], technique)).toBeNull();
+ expect(aggregateValues([null, undefined], technique)).toBeNull();
+ });
+
test('groupByDataset keeps null groups distinct from the string __null__', () => {
const dataset = makeTable({
id: 'cities',
diff --git a/src/table/src/dataset-ops/aggregations.ts b/src/table/src/dataset-ops/aggregations.ts
index ba9b94bca..36361b293 100644
--- a/src/table/src/dataset-ops/aggregations.ts
+++ b/src/table/src/dataset-ops/aggregations.ts
@@ -115,6 +115,17 @@ export function aggregateValues(values: unknown[], technique: DatasetOpAggregati
if (!defined.length) {
return null;
}
+ if (
+ technique === DATASET_OPS_AGGREGATIONS.sum ||
+ technique === DATASET_OPS_AGGREGATIONS.average ||
+ technique === DATASET_OPS_AGGREGATIONS.median
+ ) {
+ // D3 arithmetic requires Numbers. Keep raw BigInts for exact extrema/distinct counts
+ // and leave the source data unchanged; arithmetic results use Number precision.
+ return aggregate(defined, technique, value =>
+ typeof value === 'bigint' ? Number(value) : value
+ );
+ }
return aggregate(defined, technique);
}