Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix: render separated GeoArrow point coordinates (#3948)

Igor Dykhta89de37ede5de
diff --git a/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts b/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts
index 9f96e95d5..633211bfe 100644
--- a/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts
+++ b/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts
@@ -22,6 +22,7 @@ import {GEOARROW_EXTENSIONS} from '@kepler.gl/constants';
import {
assignAccessor,
+ convertStructToFixedSizeList,
extractAccessorsFromProps,
getGeometryVector,
invertOffsets
@@ -151,7 +152,7 @@ export class GeoArrowScatterplotLayer<ExtraProps extends object = object> extend
const layers: ScatterplotLayer<any>[] = [];
for (let recordBatchIdx = 0; recordBatchIdx < table.batches.length; recordBatchIdx++) {
- const geometryData = geometryColumn.data[recordBatchIdx];
+ const geometryData = convertStructToFixedSizeList(geometryColumn.data[recordBatchIdx]);
const flatCoordsData = ga.child.getPointChild(geometryData);
const flatCoordinateArray = flatCoordsData.values;
diff --git a/src/deckgl-arrow-layers/src/utils/utils.ts b/src/deckgl-arrow-layers/src/utils/utils.ts
index 7b1c194f7..90f22486f 100644
--- a/src/deckgl-arrow-layers/src/utils/utils.ts
+++ b/src/deckgl-arrow-layers/src/utils/utils.ts
@@ -8,7 +8,7 @@
import * as arrow from 'apache-arrow';
import * as ga from '@geoarrow/geoarrow-js';
import {AccessorContext, AccessorFunction, _InternalAccessorContext} from '../types';
-import {isArrowFixedSizeList, isArrowStruct, isArrowVector} from '@kepler.gl/utils';
+import {isArrowVector} from '@kepler.gl/utils';
export function assert(condition: unknown, message?: string): asserts condition {
if (!condition) throw new Error(message || 'assert');
@@ -45,63 +45,7 @@ export function isColumnReference(input: any): input is string {
return typeof input === 'string';
}
-function isDataInterleavedCoords(
- data: arrow.Data
-): data is arrow.Data<arrow.FixedSizeList<arrow.Float64>> {
- // TODO: also check 2 or 3d? Float64?
- return isArrowFixedSizeList(data.type);
-}
-
-function isDataSeparatedCoords(
- data: arrow.Data
-): data is arrow.Data<arrow.Struct<{x: arrow.Float64; y: arrow.Float64}>> {
- // TODO: also check child names? Float64?
- return isArrowStruct(data.type);
-}
-
-/**
- * Convert geoarrow Struct coordinates to FixedSizeList coords
- *
- * The GeoArrow spec allows for either separated or interleaved coords, but at
- * this time deck.gl only supports interleaved.
- */
-// TODO: this hasn't been tested yet
-export function convertStructToFixedSizeList(
- coords:
- | arrow.Data<arrow.FixedSizeList<arrow.Float64>>
- | arrow.Data<arrow.Struct<{x: arrow.Float64; y: arrow.Float64}>>
-): arrow.Data<arrow.FixedSizeList<arrow.Float64>> {
- if (isDataInterleavedCoords(coords)) {
- return coords;
- } else if (isDataSeparatedCoords(coords)) {
- // TODO: support 3d
- const interleavedCoords = new Float64Array(coords.length * 2);
- const [xChild, yChild] = coords.children;
- for (let i = 0; i < coords.length; i++) {
- interleavedCoords[i * 2] = xChild.values[i];
- interleavedCoords[i * 2 + 1] = yChild.values[i];
- }
-
- const childDataType = new arrow.Float64();
- const dataType = new arrow.FixedSizeList(2, new arrow.Field('coords', childDataType));
-
- const interleavedCoordsData = arrow.makeData({
- type: childDataType,
- length: interleavedCoords.length
- });
-
- const data = arrow.makeData({
- type: dataType,
- length: coords.length,
- nullCount: coords.nullCount,
- nullBitmap: coords.nullBitmap,
- child: interleavedCoordsData
- });
- return data;
- }
-
- assert(false);
-}
+export {convertStructToFixedSizeList} from '@kepler.gl/utils';
type AssignAccessorProps = {
/** The object on which to assign the resolved accesor */
diff --git a/src/layers/src/layer-utils.ts b/src/layers/src/layer-utils.ts
index 3fad2b155..be5feb7ae 100644
--- a/src/layers/src/layer-utils.ts
+++ b/src/layers/src/layer-utils.ts
@@ -16,7 +16,11 @@ import {
LayerColumns,
RGBColor
} from '@kepler.gl/types';
-import {DataContainerInterface, ArrowDataContainer} from '@kepler.gl/utils';
+import {
+ DataContainerInterface,
+ ArrowDataContainer,
+ convertStructToFixedSizeList
+} from '@kepler.gl/utils';
import {getBinaryGeometriesFromArrow, BinaryGeometriesFromArrowOptions} from '@loaders.gl/gis';
import {updateBoundsFromGeoArrowSamples} from '@loaders.gl/geoarrow';
import {geojsonToBinary} from '@loaders.gl/gis';
@@ -203,7 +207,9 @@ export function getGeojsonLayerMetaFromArrow({
...(chunkIndex !== undefined && chunkIndex >= 0
? {
chunkIndex,
- chunkOffset: geoColumn.data[0].length * chunkIndex
+ chunkOffset: geoColumn.data
+ .slice(0, chunkIndex)
+ .reduce((offset, chunk) => offset + chunk.length, 0)
}
: {}),
triangulate: true,
@@ -215,6 +221,28 @@ export function getGeojsonLayerMetaFromArrow({
return getBinaryGeometriesFromWKBArrow(geoColumn, options);
}
+ if (encoding === GEOARROW_EXTENSIONS.POINT) {
+ const chunks =
+ options.chunkIndex === undefined ? geoColumn.data : [geoColumn.data[options.chunkIndex]];
+ geoColumn = arrow.makeVector(
+ chunks.map(chunk => {
+ const normalized = convertStructToFixedSizeList(chunk);
+ // Interleaved chunks may belong to DuckDB's Arrow runtime. Wrap them
+ // locally for makeVector's instanceof check, retaining their buffers.
+ return new arrow.Data(
+ normalized.type,
+ normalized.offset,
+ normalized.length,
+ normalized.nullCount,
+ normalized.buffers,
+ normalized.children,
+ normalized.dictionary
+ );
+ })
+ );
+ if (options.chunkIndex !== undefined) options.chunkIndex = 0;
+ }
+
// create binary data from arrow data for GeoJsonLayer
const {binaryGeometries, featureTypes, bounds, meanCenters} = getBinaryGeometriesFromArrow(
// @ts-ignore
diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.ts
index 85ab3ee04..3b7ca18a2 100644
--- a/src/processors/src/file-handler.ts
+++ b/src/processors/src/file-handler.ts
@@ -16,7 +16,8 @@ import {
generateHashIdFromString,
getApplicationConfig,
getError,
- isArrowTable
+ isArrowTable,
+ convertStructToFixedSizeList
} from '@kepler.gl/utils';
import {generateHashId} from '@kepler.gl/common-utils';
import {
@@ -266,7 +267,10 @@ export async function parseGeoArrowBatchesOnWorkers(
? [geoColumn.data.length - 1]
: geoColumn.data.map((_, index) => index);
chunkIndexes.forEach(chunkIndex => {
- const geometryChunk = geoColumn.data[chunkIndex];
+ const geometryChunk =
+ encoding === GEOARROW_EXTENSIONS.POINT
+ ? convertStructToFixedSizeList(geoColumn.data[chunkIndex])
+ : geoColumn.data[chunkIndex];
const chunkCopy = hardClone(geometryChunk, true);
const chunkOffset = geoColumn.data
.slice(0, chunkIndex)
@@ -299,12 +303,27 @@ export async function parseGeoArrowBatchesOnWorkers(
chunkIndex,
result: parseOnWorker(sourceData),
fallback: () =>
- getBinaryGeometriesFromArrow(geoColumn, sourceData.geometryEncoding, {
- chunkIndex,
- chunkOffset,
- calculateMeanCenters: true,
- triangulate: true
- })
+ getBinaryGeometriesFromArrow(
+ // Use the original buffers: the worker copy may be transferred.
+ // A local Data wrapper also accepts chunks from another Arrow runtime.
+ arrow.makeVector(
+ new arrow.Data(
+ geometryChunk.type,
+ geometryChunk.offset,
+ geometryChunk.length,
+ geometryChunk.nullCount,
+ geometryChunk.buffers,
+ geometryChunk.children,
+ geometryChunk.dictionary
+ )
+ ),
+ sourceData.geometryEncoding,
+ {
+ chunkOffset,
+ calculateMeanCenters: true,
+ triangulate: true
+ }
+ )
});
});
});
diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.ts
index e74a235af..9169767f8 100644
--- a/src/utils/src/arrow-data-container.ts
+++ b/src/utils/src/arrow-data-container.ts
@@ -67,6 +67,58 @@ export function isArrowVector(data: any): data is arrow.Vector {
);
}
+/**
+ * Normalize native point coordinates for deck.gl and loaders.gl. Interleaved
+ * coordinates are already usable and retain their buffers without copying.
+ */
+export function convertStructToFixedSizeList(coords: arrow.Data): arrow.Data<arrow.FixedSizeList> {
+ if (arrow.DataType.isFixedSizeList(coords.type)) {
+ const pointData = coords as arrow.Data<arrow.FixedSizeList>;
+ const child = pointData.children[0];
+ const length = coords.length * coords.type.listSize;
+ // IPC/compaction can pad the final buffer. Expose only logical coordinates
+ // to loaders.gl, which derives feature counts and bounds from buffer length.
+ return child.values.length === length
+ ? pointData
+ : pointData.clone<arrow.FixedSizeList>(
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ [child.slice(0, length)]
+ );
+ }
+ if (!arrow.DataType.isStruct(coords.type)) {
+ throw new Error('Expected GeoArrow point coordinates');
+ }
+
+ const dimensions = coords.children.length;
+ const values = new Float64Array(coords.length * dimensions);
+ for (let i = 0; i < coords.length; i++) {
+ // Arrow slices already slice child values. Parent/child validity bitmaps
+ // retain their own offsets, which getValid accounts for.
+ const valid =
+ coords.getValid(i) &&
+ coords.children.every(child => child.getValid(i) && Number.isFinite(child.values[i]));
+ for (let axis = 0; axis < dimensions; axis++) {
+ // Preserve row alignment while keeping missing/empty points out of bounds
+ // and rendering. Float64 also preserves Float32 input values exactly.
+ values[i * dimensions + axis] = valid ? coords.children[axis].values[i] : NaN;
+ }
+ }
+
+ const floatType = new arrow.Float64();
+ return arrow.makeData({
+ type: new arrow.FixedSizeList(dimensions, new arrow.Field('coords', floatType)),
+ offset: coords.offset,
+ length: coords.length,
+ nullCount: coords.nullCount,
+ nullBitmap: coords.nullBitmap,
+ child: arrow.makeData({type: floatType, length: values.length, data: values})
+ });
+}
+
/** Build an Arrow vector while letting Arrow infer the physical type from converted values. */
export function createArrowVector(values: unknown[]): arrow.Vector {
return arrow.vectorFromArray(values as any[]);
diff --git a/src/utils/src/index.ts b/src/utils/src/index.ts
index 32f24d29b..6e966cecc 100644
--- a/src/utils/src/index.ts
+++ b/src/utils/src/index.ts
@@ -168,6 +168,7 @@ export * from './map-utils';
export {
ArrowDataContainer,
createArrowVector,
+ convertStructToFixedSizeList,
compactArrowTable,
arrowDataTypeToAnalyzerDataType,
arrowDataTypeToFieldType,
diff --git a/test/browser/layer-tests/point-layer-specs.js b/test/browser/layer-tests/point-layer-specs.js
index 7963949cb..dfbae035f 100644
--- a/test/browser/layer-tests/point-layer-specs.js
+++ b/test/browser/layer-tests/point-layer-specs.js
@@ -13,7 +13,12 @@ import {EnhancedMultiIconLayer, EnhancedTextBackgroundLayer} from '@kepler.gl/de
import {GeoArrowScatterplotLayer} from '@kepler.gl/deckgl-arrow-layers';
import {KeplerGlLayers} from '@kepler.gl/layers';
import {arrowSchemaToFields, processArrowBatches, processGeojson} from '@kepler.gl/processors';
-import {INITIAL_MAP_STATE, INITIAL_VIS_STATE, renderDeckGlLayer} from '@kepler.gl/reducers';
+import {
+ findDefaultLayer,
+ INITIAL_MAP_STATE,
+ INITIAL_VIS_STATE,
+ renderDeckGlLayer
+} from '@kepler.gl/reducers';
import {copyTableAndUpdate, KeplerTable} from '@kepler.gl/table';
import {hexToRgb} from '@kepler.gl/utils';
@@ -37,6 +42,89 @@ import {
const {PointLayer} = KeplerGlLayers;
+test('#PointLayer -> native separated points preserve every coordinate through import', async t => {
+ for (const dimensions of [2, 3]) {
+ for (const separated of [true, false]) {
+ const names = ['x', 'y', 'z'].slice(0, dimensions);
+ const type = separated
+ ? new arrow.Struct(names.map(name => new arrow.Field(name, new arrow.Float64(), false)))
+ : new arrow.FixedSizeList(dimensions, new arrow.Field('coords', new arrow.Float64()));
+ const coordinates = [
+ [10, 20, 5],
+ [30, 40, 15],
+ [50, 60, 25]
+ ].map(p => p.slice(0, dimensions));
+ const batches = [coordinates.slice(0, 2), coordinates.slice(2)].map(points => {
+ const table = new arrow.Table({
+ geometry: arrow.vectorFromArray(
+ separated
+ ? points.map(point => Object.fromEntries(names.map((name, i) => [name, point[i]])))
+ : points,
+ type
+ )
+ });
+ table.schema.fields[0].metadata.set('ARROW:extension:name', 'geoarrow.point');
+ return table;
+ });
+ const table = arrow.tableFromIPC(arrow.tableToIPC(batches[0].concat(batches[1]), 'file'));
+ const originalBytes = arrow.tableToIPC(table, 'file');
+ const dataset = new KeplerTable({info: {id: 'native-points'}, color: [0, 0, 0]});
+ await dataset.importData({data: processArrowBatches(table.batches)});
+ const [geojson] = findDefaultLayer(dataset, {geojson: KeplerGlLayers.GeojsonLayer});
+ t.ok(geojson, 'native point metadata discovers the default GeoJSON layer');
+ geojson.formatLayerData({[dataset.id]: dataset});
+ t.deepEqual(
+ Array.from(geojson.dataToFeature[0].points.positions.value),
+ coordinates.flat(),
+ 'default GeoJSON positions contain every axis'
+ );
+ t.equal(geojson.dataToFeature[0].points.positions.size, dimensions, 'GeoJSON position size');
+ t.deepEqual(
+ geojson.meta.bounds,
+ [10, 20, 50, 60],
+ 'GeoJSON bounds use longitude and latitude'
+ );
+
+ // Progressive imports retain unequal record batches, including Arrow slices.
+ const sliced = table.slice(1);
+ await dataset.update({cols: [sliced.getChildAt(0)], arrowTable: sliced});
+ const layer = new PointLayer({dataId: dataset.id, columnMode: 'geoarrow'});
+ layer.config.columns.geoarrow = {value: 'geometry', fieldIdx: 0};
+ const formatted = layer.formatLayerData({[dataset.id]: dataset});
+ const [composite] = layer.renderLayer({
+ data: formatted,
+ dataset,
+ idx: 0,
+ layerInteraction: {},
+ mapState: INITIAL_MAP_STATE,
+ gpuFilter: dataset.gpuFilter,
+ interactionConfig: INITIAL_VIS_STATE.interactionConfig,
+ visible: true,
+ layerCallbacks: {}
+ });
+ const sublayers = composite.renderLayers();
+ t.equal(sublayers.length, 2, 'Point retains both sliced batches');
+ sublayers.forEach((sublayer, i) => {
+ const position = sublayer.props.data.attributes.getPosition;
+ t.deepEqual(
+ Array.from(position.value),
+ coordinates[i + 1],
+ 'Point positions include latitude and altitude'
+ );
+ t.equal(position.size, dimensions, 'Point position size');
+ t.equal(
+ composite.getPickingInfo({info: {index: 0}, sourceLayer: sublayer}).index,
+ i,
+ 'picking remains aligned'
+ );
+ });
+ t.deepEqual(layer.meta.bounds, [30, 40, 50, 60], 'Point bounds reflect the slice');
+ t.deepEqual(arrow.tableToIPC(table, 'file'), originalBytes, 'rendering preserves the source');
+ }
+ }
+ t.end();
+});
+
['point', 'multipoint'].forEach(geometryType => {
[false, true].forEach(gpu => {
test(`#PointLayer -> batched Arrow ${geometryType} accessors (${
diff --git a/test/node/processors/file-handler-test.js b/test/node/processors/file-handler-test.js
index 5068bcff8..37ccb463d 100644
--- a/test/node/processors/file-handler-test.js
+++ b/test/node/processors/file-handler-test.js
@@ -25,6 +25,8 @@ import {
getDatasetRefreshIntervalMs
} from '@kepler.gl/constants';
import * as arrow from 'apache-arrow';
+// The standalone bundle has distinct constructors, like DuckDB's bundled Arrow.
+import * as foreignArrow from 'apache-arrow/Arrow.es2015.min';
import {convertArrowToSchema} from '@loaders.gl/schema-utils';
import {parsedFields, parsedRows} from 'test/fixtures/row-object';
import {
@@ -515,6 +517,135 @@ test('#file-handler -> parseGeoArrowBatchesOnWorkers parses the latest chunk', a
t.end();
});
+test('#file-handler -> separated points reach workers and fallback with all coordinates', async t => {
+ const type = new arrow.Struct(
+ ['x', 'y', 'z'].map(name => new arrow.Field(name, new arrow.Float64(), false))
+ );
+ const batches = [
+ [
+ {x: 10, y: 20, z: 5},
+ {x: 30, y: 40, z: 15}
+ ],
+ [{x: 50, y: 60, z: 25}]
+ ].map(points => {
+ const table = new arrow.Table({geometry: arrow.vectorFromArray(points, type)});
+ table.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT);
+ return table;
+ });
+ const table = arrow.tableFromIPC(arrow.tableToIPC(batches[0].concat(batches[1]), 'file'));
+ const bytes = arrow.tableToIPC(table, 'file');
+ const processed = processArrowBatches(table.batches, {compact: false});
+ for (const failWorker of [false, true]) {
+ const result = await parseGeoArrowBatchesOnWorkers(processed, {
+ parseOnWorker: async input => {
+ t.equal(input.chunkData.type.listSize, 3, 'worker input has XYZ coordinate size');
+ if (failWorker) throw new Error('worker unavailable');
+ return parseGeoArrowWorkerInput(input);
+ }
+ });
+ const {layer} = await layerFromFile({
+ info: {id: 'separated'},
+ data: {...processed, geoArrowBinaryData: result}
+ });
+ t.deepEqual(
+ layer.dataToFeature.flatMap(data => Array.from(data.points.positions.value)),
+ [10, 20, 5, 30, 40, 15, 50, 60, 25],
+ 'worker/cache and fallback retain all axes'
+ );
+ t.deepEqual(
+ layer.meta.bounds,
+ [10, 20, 50, 60],
+ 'cached bounds include latitude from every batch'
+ );
+ t.deepEqual(
+ layer.dataToFeature.flatMap(data => Array.from(data.points.globalFeatureIds.value)),
+ [0, 1, 2],
+ 'cached feature ids retain unequal batch offsets'
+ );
+ }
+ t.deepEqual(arrow.tableToIPC(table, 'file'), bytes, 'worker preparation preserves source data');
+ t.end();
+});
+
+test('#file-handler -> worker fallback accepts points and lines from another Arrow runtime', async t => {
+ const pointType = new foreignArrow.FixedSizeList(
+ 2,
+ new foreignArrow.Field('xy', new foreignArrow.Float64())
+ );
+ for (const encoding of [GEOARROW_EXTENSIONS.POINT, GEOARROW_EXTENSIONS.LINESTRING]) {
+ const isPoint = encoding === GEOARROW_EXTENSIONS.POINT;
+ const chunks = [
+ [
+ [10, 20],
+ [30, 40]
+ ],
+ [[50, 60]]
+ ].map(points => {
+ // Exact child/offset lengths keep unrelated IPC padding out of line fixtures.
+ const coordinates = isPoint ? points : points.flatMap(point => [point, point]);
+ const child = foreignArrow
+ .vectorFromArray(coordinates, pointType)
+ .data[0].slice(0, coordinates.length);
+ return isPoint
+ ? child
+ : foreignArrow.makeData({
+ type: new foreignArrow.List(new foreignArrow.Field('vertices', pointType)),
+ length: points.length,
+ valueOffsets: Int32Array.from(points.map((_, i) => i * 2).concat(points.length * 2)),
+ child
+ });
+ });
+ const geoColumn = foreignArrow.makeVector(chunks);
+ const field = {
+ name: 'geometry',
+ type: 'geoarrow',
+ fieldIdx: 0,
+ metadata: new Map([[GEOARROW_METADATA_KEY, encoding]])
+ };
+ const table = new foreignArrow.Table({geometry: geoColumn});
+ const originalBytes = foreignArrow.tableToIPC(table);
+ t.notOk(chunks[0] instanceof arrow.Data, 'fixture uses a separate Arrow runtime');
+
+ for (const latestOnly of [false, true]) {
+ const result = await parseGeoArrowBatchesOnWorkers(
+ {fields: [field], cols: [geoColumn]},
+ {
+ latestOnly,
+ parseOnWorker: async () => {
+ throw new Error('worker unavailable');
+ }
+ }
+ );
+ t.deepEqual(Object.keys(result[0]), latestOnly ? ['1'] : ['0', '1'], 'selected batches');
+ for (const [chunkIndex, binary] of Object.entries(result[0])) {
+ const last = chunkIndex === '1';
+ const points = last
+ ? [[50, 60]]
+ : [
+ [10, 20],
+ [30, 40]
+ ];
+ const rowIds = last ? [2] : [0, 1];
+ const geometry = binary.binaryGeometries[0][isPoint ? 'points' : 'lines'];
+ t.deepEqual(
+ Array.from(geometry.positions.value),
+ (isPoint ? points : points.flatMap(point => [point, point])).flat(),
+ 'fallback preserves all coordinates'
+ );
+ t.deepEqual(
+ Array.from(geometry.globalFeatureIds.value),
+ isPoint ? rowIds : rowIds.flatMap(id => [id, id]),
+ 'fallback preserves global row ids'
+ );
+ t.deepEqual(binary.bounds, last ? [50, 60, 50, 60] : [10, 20, 30, 40], 'bounds');
+ t.deepEqual(binary.meanCenters, points, 'centroids remain aligned with rows');
+ }
+ }
+ t.deepEqual(foreignArrow.tableToIPC(table), originalBytes, 'fallback preserves source data');
+ }
+ t.end();
+});
+
test('#file-handler -> parseGeoArrowBatchesOnWorkers combines compacted worker chunks', async t => {
const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false));
const batchA = new arrow.Table({
diff --git a/test/node/utils/layer-utils-test.js b/test/node/utils/layer-utils-test.js
index 09955e878..5cf571f4f 100644
--- a/test/node/utils/layer-utils-test.js
+++ b/test/node/utils/layer-utils-test.js
@@ -30,6 +30,10 @@ import {
findDefaultColorFieldWithValues
} from '@kepler.gl/utils';
import * as arrow from 'apache-arrow';
+// The standalone bundle has distinct constructors, like DuckDB's bundled Arrow.
+import * as foreignArrow from 'apache-arrow/Arrow.es2015.min';
+import {convertStructToFixedSizeList} from '../../../src/deckgl-arrow-layers/src/utils/utils';
+import {GeoArrowScatterplotLayer} from '@kepler.gl/deckgl-arrow-layers';
const defaultColorLayerTypes = ['a5', 's2', 'geohash', 'hexagonId', 'point', 'icon'];
@@ -278,6 +282,223 @@ test('layerUtils -> createGeoArrowPointVector handles empty and nonempty coordin
t.end();
});
+test('layerUtils -> separated point normalization preserves validity, slices and precision', t => {
+ for (const FloatType of [arrow.Float32, arrow.Float64]) {
+ for (const dimensions of [2, 3]) {
+ const names = ['x', 'y', 'z'].slice(0, dimensions);
+ const type = new arrow.Struct(names.map(name => new arrow.Field(name, new FloatType())));
+ const vector = arrow
+ .vectorFromArray(
+ [
+ {x: -100, y: -80, z: -1},
+ null,
+ {x: 10.123456789, y: 20.123456789, z: 5.123456789},
+ {x: NaN, y: NaN, z: NaN},
+ {x: Infinity, y: 40, z: 1},
+ {x: 50, y: null, z: 1},
+ {x: 30, y: 40, z: 15}
+ ],
+ type
+ )
+ .slice(1);
+ const data = vector.data[0];
+ const normalized = convertStructToFixedSizeList(data);
+ const validPoint = names.map(name => vector.get(1)[name]);
+ const expected = [
+ ...Array(dimensions).fill(NaN),
+ ...validPoint,
+ ...Array(3 * dimensions).fill(NaN),
+ ...names.map(name => vector.get(5)[name])
+ ];
+ t.deepEqual(
+ Array.from(normalized.children[0].values),
+ expected,
+ 'all axes retain source precision; missing and nonfinite rows are NaN'
+ );
+ t.equal(arrow.makeVector(normalized).get(0), null, 'slice retains parent null validity');
+ t.deepEqual(
+ Array.from(arrow.makeVector(normalized).get(1)),
+ validPoint,
+ 'slice retains valid row values'
+ );
+ t.ok(
+ normalized.children[0].values instanceof Float64Array,
+ 'does not narrow coordinate precision'
+ );
+
+ const metadata = new Map([[GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT]]);
+ const geoField = {name: 'geometry', type: 'geoarrow', fieldIdx: 0, metadata};
+ const dataContainer = new ArrowDataContainer({cols: [vector], fields: [geoField]});
+ const meta = getGeojsonLayerMetaFromArrow({
+ dataContainer,
+ geoColumn: vector,
+ geoField,
+ columnIndex: 0
+ });
+ t.deepEqual(
+ Array.from(meta.dataToFeature[0].points.positions.value),
+ expected,
+ 'GeoJSON uses normalized rows'
+ );
+ t.deepEqual(
+ meta.bounds,
+ [validPoint[0], validPoint[1], 30, 40],
+ 'null and nonfinite rows do not distort bounds'
+ );
+ const table = new arrow.Table({geometry: vector});
+ table.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT);
+ const [layer] = new GeoArrowScatterplotLayer({data: table}).renderLayers();
+ t.deepEqual(
+ Array.from(layer.props.data.attributes.getPosition.value),
+ expected,
+ 'Point metadata inference uses normalized rows'
+ );
+ t.equal(layer.props.data.length, vector.length, 'invalid rows retain picking alignment');
+ }
+ }
+ const type = new arrow.Struct(['x', 'y'].map(name => new arrow.Field(name, new arrow.Float64())));
+ for (const rows of [[], [null, null]]) {
+ const input = arrow.vectorFromArray(rows, type).data[0];
+ const output = convertStructToFixedSizeList(input);
+ t.equal(output.length, rows.length, 'empty/all-null input keeps its row count');
+ t.equal(output.type.listSize, 2, 'empty/all-null coordinates still have two dimensions');
+ t.ok(
+ Array.from(output.children[0].values).every(Number.isNaN),
+ 'missing points never become the origin'
+ );
+ }
+ for (const FloatType of [arrow.Float32, arrow.Float64]) {
+ const type = new arrow.FixedSizeList(2, new arrow.Field('xy', new FloatType()));
+ const input = arrow
+ .vectorFromArray(
+ [
+ [0, 0],
+ [10, 20],
+ [30, 40]
+ ],
+ type
+ )
+ .slice(1).data[0];
+ t.equal(
+ convertStructToFixedSizeList(input),
+ input,
+ 'interleaved slices retain the zero-copy fast path'
+ );
+ }
+ t.end();
+});
+
+test('layerUtils -> incremental separated points retain unequal batch row offsets', t => {
+ const type = new arrow.Struct(['x', 'y'].map(name => new arrow.Field(name, new arrow.Float64())));
+ const chunks = [
+ [
+ {x: 10, y: 20},
+ {x: 30, y: 40}
+ ],
+ [{x: 50, y: 60}],
+ [{x: 70, y: 80}]
+ ].map(rows => arrow.vectorFromArray(rows, type).data[0]);
+ const geoColumn = arrow.makeVector(chunks);
+ const metadata = new Map([[GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT]]);
+ const geoField = {name: 'geometry', type: 'geoarrow', fieldIdx: 0, metadata};
+ const dataContainer = new ArrowDataContainer({cols: [geoColumn], fields: [geoField]});
+ const result = getGeojsonLayerMetaFromArrow({
+ dataContainer,
+ geoColumn,
+ geoField,
+ columnIndex: 0,
+ chunkIndex: 2
+ });
+ t.equal(result.dataToFeature.length, 1, 'only converts the requested batch');
+ t.deepEqual(
+ Array.from(result.dataToFeature[0].points.positions.value),
+ [70, 80],
+ 'retains the requested coordinates'
+ );
+ t.deepEqual(
+ Array.from(result.dataToFeature[0].points.globalFeatureIds.value),
+ [3],
+ 'sums preceding unequal batch lengths'
+ );
+ t.deepEqual(result.centroids, [[70, 80]], 'centroid uses both coordinate axes');
+ t.end();
+});
+
+test('layerUtils -> native points from another Arrow runtime preserve slices and batch ids', t => {
+ const type = new foreignArrow.FixedSizeList(
+ 3,
+ new foreignArrow.Field('xyz', new foreignArrow.Float64())
+ );
+ const first = foreignArrow
+ .vectorFromArray(
+ [
+ [-100, -80, -1],
+ [10, 20, 5],
+ [30, 40, 15]
+ ],
+ type
+ )
+ .slice(1);
+ const last = foreignArrow.vectorFromArray([[50, 60, 25]], type);
+ const geoColumn = foreignArrow.makeVector([...first.data, ...last.data]);
+ const geoField = {
+ name: 'geometry',
+ type: 'geoarrow',
+ fieldIdx: 0,
+ metadata: new Map([[GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT]])
+ };
+ const table = new foreignArrow.Table({geometry: geoColumn});
+ const originalBytes = foreignArrow.tableToIPC(table);
+ const dataContainer = new ArrowDataContainer({
+ cols: [geoColumn],
+ fields: [geoField],
+ arrowTable: table
+ });
+ t.notOk(geoColumn.data[0] instanceof arrow.Data, 'fixture uses a separate Arrow runtime');
+
+ for (const chunkIndex of [undefined, 1]) {
+ const result = getGeojsonLayerMetaFromArrow({
+ dataContainer,
+ geoColumn,
+ geoField,
+ columnIndex: 0,
+ chunkIndex
+ });
+ const incremental = chunkIndex === 1;
+ t.deepEqual(
+ result.dataToFeature.flatMap(data => Array.from(data.points.positions.value)),
+ incremental ? [50, 60, 25] : [10, 20, 5, 30, 40, 15, 50, 60, 25],
+ 'preserves XYZ coordinates and excludes padding and sliced-out rows'
+ );
+ t.deepEqual(
+ result.dataToFeature.flatMap(data => Array.from(data.points.globalFeatureIds.value)),
+ incremental ? [2] : [0, 1, 2],
+ 'retains row ids across unequal batches'
+ );
+ t.deepEqual(result.bounds, incremental ? [50, 60, 50, 60] : [10, 20, 50, 60], 'bounds');
+ t.deepEqual(
+ result.centroids,
+ incremental
+ ? [[50, 60]]
+ : [
+ [10, 20],
+ [30, 40],
+ [50, 60]
+ ],
+ 'centroids remain aligned with rows'
+ );
+ result.dataToFeature.forEach((data, i) => {
+ t.equal(
+ data.points.positions.value.buffer,
+ geoColumn.data[(chunkIndex || 0) + i].children[0].values.buffer,
+ 'interleaved coordinates still share their source buffer'
+ );
+ });
+ }
+ t.deepEqual(foreignArrow.tableToIPC(table), originalBytes, 'does not change the source table');
+ t.end();
+});
+
test('layerUtils -> getGeojsonLayerMetaFromArrow uses worker-prepared data', t => {
const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false));
const geoColumn = arrow.vectorFromArray([[-122.4, 37.8]], pointType);