Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix: render separated GeoArrow point coordinates (#3948)
Igor Dykhta89de37ede5de
diff --git a/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts b/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.tsindex 9f96e95d5..633211bfe 100644--- a/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts+++ b/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts@@ -22,6 +22,7 @@ import {GEOARROW_EXTENSIONS} from '@kepler.gl/constants';import {assignAccessor,+ convertStructToFixedSizeList,extractAccessorsFromProps,getGeometryVector,invertOffsets@@ -151,7 +152,7 @@ export class GeoArrowScatterplotLayer<ExtraProps extends object = object> extendconst layers: ScatterplotLayer<any>[] = [];for (let recordBatchIdx = 0; recordBatchIdx < table.batches.length; recordBatchIdx++) {- const geometryData = geometryColumn.data[recordBatchIdx];+ const geometryData = convertStructToFixedSizeList(geometryColumn.data[recordBatchIdx]);const flatCoordsData = ga.child.getPointChild(geometryData);const flatCoordinateArray = flatCoordsData.values;diff --git a/src/deckgl-arrow-layers/src/utils/utils.ts b/src/deckgl-arrow-layers/src/utils/utils.tsindex 7b1c194f7..90f22486f 100644--- a/src/deckgl-arrow-layers/src/utils/utils.ts+++ b/src/deckgl-arrow-layers/src/utils/utils.ts@@ -8,7 +8,7 @@import * as arrow from 'apache-arrow';import * as ga from '@geoarrow/geoarrow-js';import {AccessorContext, AccessorFunction, _InternalAccessorContext} from '../types';-import {isArrowFixedSizeList, isArrowStruct, isArrowVector} from '@kepler.gl/utils';+import {isArrowVector} from '@kepler.gl/utils';export function assert(condition: unknown, message?: string): asserts condition {if (!condition) throw new Error(message || 'assert');@@ -45,63 +45,7 @@ export function isColumnReference(input: any): input is string {return typeof input === 'string';}-function isDataInterleavedCoords(- data: arrow.Data-): data is arrow.Data<arrow.FixedSizeList<arrow.Float64>> {- // TODO: also check 2 or 3d? Float64?- return isArrowFixedSizeList(data.type);-}--function isDataSeparatedCoords(- data: arrow.Data-): data is arrow.Data<arrow.Struct<{x: arrow.Float64; y: arrow.Float64}>> {- // TODO: also check child names? Float64?- return isArrowStruct(data.type);-}--/**- * Convert geoarrow Struct coordinates to FixedSizeList coords- *- * The GeoArrow spec allows for either separated or interleaved coords, but at- * this time deck.gl only supports interleaved.- */-// TODO: this hasn't been tested yet-export function convertStructToFixedSizeList(- coords:- | arrow.Data<arrow.FixedSizeList<arrow.Float64>>- | arrow.Data<arrow.Struct<{x: arrow.Float64; y: arrow.Float64}>>-): arrow.Data<arrow.FixedSizeList<arrow.Float64>> {- if (isDataInterleavedCoords(coords)) {- return coords;- } else if (isDataSeparatedCoords(coords)) {- // TODO: support 3d- const interleavedCoords = new Float64Array(coords.length * 2);- const [xChild, yChild] = coords.children;- for (let i = 0; i < coords.length; i++) {- interleavedCoords[i * 2] = xChild.values[i];- interleavedCoords[i * 2 + 1] = yChild.values[i];- }-- const childDataType = new arrow.Float64();- const dataType = new arrow.FixedSizeList(2, new arrow.Field('coords', childDataType));-- const interleavedCoordsData = arrow.makeData({- type: childDataType,- length: interleavedCoords.length- });-- const data = arrow.makeData({- type: dataType,- length: coords.length,- nullCount: coords.nullCount,- nullBitmap: coords.nullBitmap,- child: interleavedCoordsData- });- return data;- }-- assert(false);-}+export {convertStructToFixedSizeList} from '@kepler.gl/utils';type AssignAccessorProps = {/** The object on which to assign the resolved accesor */diff --git a/src/layers/src/layer-utils.ts b/src/layers/src/layer-utils.tsindex 3fad2b155..be5feb7ae 100644--- a/src/layers/src/layer-utils.ts+++ b/src/layers/src/layer-utils.ts@@ -16,7 +16,11 @@ import {LayerColumns,RGBColor} from '@kepler.gl/types';-import {DataContainerInterface, ArrowDataContainer} from '@kepler.gl/utils';+import {+ DataContainerInterface,+ ArrowDataContainer,+ convertStructToFixedSizeList+} from '@kepler.gl/utils';import {getBinaryGeometriesFromArrow, BinaryGeometriesFromArrowOptions} from '@loaders.gl/gis';import {updateBoundsFromGeoArrowSamples} from '@loaders.gl/geoarrow';import {geojsonToBinary} from '@loaders.gl/gis';@@ -203,7 +207,9 @@ export function getGeojsonLayerMetaFromArrow({...(chunkIndex !== undefined && chunkIndex >= 0? {chunkIndex,- chunkOffset: geoColumn.data[0].length * chunkIndex+ chunkOffset: geoColumn.data+ .slice(0, chunkIndex)+ .reduce((offset, chunk) => offset + chunk.length, 0)}: {}),triangulate: true,@@ -215,6 +221,28 @@ export function getGeojsonLayerMetaFromArrow({return getBinaryGeometriesFromWKBArrow(geoColumn, options);}+ if (encoding === GEOARROW_EXTENSIONS.POINT) {+ const chunks =+ options.chunkIndex === undefined ? geoColumn.data : [geoColumn.data[options.chunkIndex]];+ geoColumn = arrow.makeVector(+ chunks.map(chunk => {+ const normalized = convertStructToFixedSizeList(chunk);+ // Interleaved chunks may belong to DuckDB's Arrow runtime. Wrap them+ // locally for makeVector's instanceof check, retaining their buffers.+ return new arrow.Data(+ normalized.type,+ normalized.offset,+ normalized.length,+ normalized.nullCount,+ normalized.buffers,+ normalized.children,+ normalized.dictionary+ );+ })+ );+ if (options.chunkIndex !== undefined) options.chunkIndex = 0;+ }+// create binary data from arrow data for GeoJsonLayerconst {binaryGeometries, featureTypes, bounds, meanCenters} = getBinaryGeometriesFromArrow(// @ts-ignorediff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.tsindex 85ab3ee04..3b7ca18a2 100644--- a/src/processors/src/file-handler.ts+++ b/src/processors/src/file-handler.ts@@ -16,7 +16,8 @@ import {generateHashIdFromString,getApplicationConfig,getError,- isArrowTable+ isArrowTable,+ convertStructToFixedSizeList} from '@kepler.gl/utils';import {generateHashId} from '@kepler.gl/common-utils';import {@@ -266,7 +267,10 @@ export async function parseGeoArrowBatchesOnWorkers(? [geoColumn.data.length - 1]: geoColumn.data.map((_, index) => index);chunkIndexes.forEach(chunkIndex => {- const geometryChunk = geoColumn.data[chunkIndex];+ const geometryChunk =+ encoding === GEOARROW_EXTENSIONS.POINT+ ? convertStructToFixedSizeList(geoColumn.data[chunkIndex])+ : geoColumn.data[chunkIndex];const chunkCopy = hardClone(geometryChunk, true);const chunkOffset = geoColumn.data.slice(0, chunkIndex)@@ -299,12 +303,27 @@ export async function parseGeoArrowBatchesOnWorkers(chunkIndex,result: parseOnWorker(sourceData),fallback: () =>- getBinaryGeometriesFromArrow(geoColumn, sourceData.geometryEncoding, {- chunkIndex,- chunkOffset,- calculateMeanCenters: true,- triangulate: true- })+ getBinaryGeometriesFromArrow(+ // Use the original buffers: the worker copy may be transferred.+ // A local Data wrapper also accepts chunks from another Arrow runtime.+ arrow.makeVector(+ new arrow.Data(+ geometryChunk.type,+ geometryChunk.offset,+ geometryChunk.length,+ geometryChunk.nullCount,+ geometryChunk.buffers,+ geometryChunk.children,+ geometryChunk.dictionary+ )+ ),+ sourceData.geometryEncoding,+ {+ chunkOffset,+ calculateMeanCenters: true,+ triangulate: true+ }+ )});});});diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.tsindex e74a235af..9169767f8 100644--- a/src/utils/src/arrow-data-container.ts+++ b/src/utils/src/arrow-data-container.ts@@ -67,6 +67,58 @@ export function isArrowVector(data: any): data is arrow.Vector {);}+/**+ * Normalize native point coordinates for deck.gl and loaders.gl. Interleaved+ * coordinates are already usable and retain their buffers without copying.+ */+export function convertStructToFixedSizeList(coords: arrow.Data): arrow.Data<arrow.FixedSizeList> {+ if (arrow.DataType.isFixedSizeList(coords.type)) {+ const pointData = coords as arrow.Data<arrow.FixedSizeList>;+ const child = pointData.children[0];+ const length = coords.length * coords.type.listSize;+ // IPC/compaction can pad the final buffer. Expose only logical coordinates+ // to loaders.gl, which derives feature counts and bounds from buffer length.+ return child.values.length === length+ ? pointData+ : pointData.clone<arrow.FixedSizeList>(+ undefined,+ undefined,+ undefined,+ undefined,+ undefined,+ [child.slice(0, length)]+ );+ }+ if (!arrow.DataType.isStruct(coords.type)) {+ throw new Error('Expected GeoArrow point coordinates');+ }++ const dimensions = coords.children.length;+ const values = new Float64Array(coords.length * dimensions);+ for (let i = 0; i < coords.length; i++) {+ // Arrow slices already slice child values. Parent/child validity bitmaps+ // retain their own offsets, which getValid accounts for.+ const valid =+ coords.getValid(i) &&+ coords.children.every(child => child.getValid(i) && Number.isFinite(child.values[i]));+ for (let axis = 0; axis < dimensions; axis++) {+ // Preserve row alignment while keeping missing/empty points out of bounds+ // and rendering. Float64 also preserves Float32 input values exactly.+ values[i * dimensions + axis] = valid ? coords.children[axis].values[i] : NaN;+ }+ }++ const floatType = new arrow.Float64();+ return arrow.makeData({+ type: new arrow.FixedSizeList(dimensions, new arrow.Field('coords', floatType)),+ offset: coords.offset,+ length: coords.length,+ nullCount: coords.nullCount,+ nullBitmap: coords.nullBitmap,+ child: arrow.makeData({type: floatType, length: values.length, data: values})+ });+}+/** Build an Arrow vector while letting Arrow infer the physical type from converted values. */export function createArrowVector(values: unknown[]): arrow.Vector {return arrow.vectorFromArray(values as any[]);diff --git a/src/utils/src/index.ts b/src/utils/src/index.tsindex 32f24d29b..6e966cecc 100644--- a/src/utils/src/index.ts+++ b/src/utils/src/index.ts@@ -168,6 +168,7 @@ export * from './map-utils';export {ArrowDataContainer,createArrowVector,+ convertStructToFixedSizeList,compactArrowTable,arrowDataTypeToAnalyzerDataType,arrowDataTypeToFieldType,diff --git a/test/browser/layer-tests/point-layer-specs.js b/test/browser/layer-tests/point-layer-specs.jsindex 7963949cb..dfbae035f 100644--- a/test/browser/layer-tests/point-layer-specs.js+++ b/test/browser/layer-tests/point-layer-specs.js@@ -13,7 +13,12 @@ import {EnhancedMultiIconLayer, EnhancedTextBackgroundLayer} from '@kepler.gl/deimport {GeoArrowScatterplotLayer} from '@kepler.gl/deckgl-arrow-layers';import {KeplerGlLayers} from '@kepler.gl/layers';import {arrowSchemaToFields, processArrowBatches, processGeojson} from '@kepler.gl/processors';-import {INITIAL_MAP_STATE, INITIAL_VIS_STATE, renderDeckGlLayer} from '@kepler.gl/reducers';+import {+ findDefaultLayer,+ INITIAL_MAP_STATE,+ INITIAL_VIS_STATE,+ renderDeckGlLayer+} from '@kepler.gl/reducers';import {copyTableAndUpdate, KeplerTable} from '@kepler.gl/table';import {hexToRgb} from '@kepler.gl/utils';@@ -37,6 +42,89 @@ import {const {PointLayer} = KeplerGlLayers;+test('#PointLayer -> native separated points preserve every coordinate through import', async t => {+ for (const dimensions of [2, 3]) {+ for (const separated of [true, false]) {+ const names = ['x', 'y', 'z'].slice(0, dimensions);+ const type = separated+ ? new arrow.Struct(names.map(name => new arrow.Field(name, new arrow.Float64(), false)))+ : new arrow.FixedSizeList(dimensions, new arrow.Field('coords', new arrow.Float64()));+ const coordinates = [+ [10, 20, 5],+ [30, 40, 15],+ [50, 60, 25]+ ].map(p => p.slice(0, dimensions));+ const batches = [coordinates.slice(0, 2), coordinates.slice(2)].map(points => {+ const table = new arrow.Table({+ geometry: arrow.vectorFromArray(+ separated+ ? points.map(point => Object.fromEntries(names.map((name, i) => [name, point[i]])))+ : points,+ type+ )+ });+ table.schema.fields[0].metadata.set('ARROW:extension:name', 'geoarrow.point');+ return table;+ });+ const table = arrow.tableFromIPC(arrow.tableToIPC(batches[0].concat(batches[1]), 'file'));+ const originalBytes = arrow.tableToIPC(table, 'file');+ const dataset = new KeplerTable({info: {id: 'native-points'}, color: [0, 0, 0]});+ await dataset.importData({data: processArrowBatches(table.batches)});+ const [geojson] = findDefaultLayer(dataset, {geojson: KeplerGlLayers.GeojsonLayer});+ t.ok(geojson, 'native point metadata discovers the default GeoJSON layer');+ geojson.formatLayerData({[dataset.id]: dataset});+ t.deepEqual(+ Array.from(geojson.dataToFeature[0].points.positions.value),+ coordinates.flat(),+ 'default GeoJSON positions contain every axis'+ );+ t.equal(geojson.dataToFeature[0].points.positions.size, dimensions, 'GeoJSON position size');+ t.deepEqual(+ geojson.meta.bounds,+ [10, 20, 50, 60],+ 'GeoJSON bounds use longitude and latitude'+ );++ // Progressive imports retain unequal record batches, including Arrow slices.+ const sliced = table.slice(1);+ await dataset.update({cols: [sliced.getChildAt(0)], arrowTable: sliced});+ const layer = new PointLayer({dataId: dataset.id, columnMode: 'geoarrow'});+ layer.config.columns.geoarrow = {value: 'geometry', fieldIdx: 0};+ const formatted = layer.formatLayerData({[dataset.id]: dataset});+ const [composite] = layer.renderLayer({+ data: formatted,+ dataset,+ idx: 0,+ layerInteraction: {},+ mapState: INITIAL_MAP_STATE,+ gpuFilter: dataset.gpuFilter,+ interactionConfig: INITIAL_VIS_STATE.interactionConfig,+ visible: true,+ layerCallbacks: {}+ });+ const sublayers = composite.renderLayers();+ t.equal(sublayers.length, 2, 'Point retains both sliced batches');+ sublayers.forEach((sublayer, i) => {+ const position = sublayer.props.data.attributes.getPosition;+ t.deepEqual(+ Array.from(position.value),+ coordinates[i + 1],+ 'Point positions include latitude and altitude'+ );+ t.equal(position.size, dimensions, 'Point position size');+ t.equal(+ composite.getPickingInfo({info: {index: 0}, sourceLayer: sublayer}).index,+ i,+ 'picking remains aligned'+ );+ });+ t.deepEqual(layer.meta.bounds, [30, 40, 50, 60], 'Point bounds reflect the slice');+ t.deepEqual(arrow.tableToIPC(table, 'file'), originalBytes, 'rendering preserves the source');+ }+ }+ t.end();+});+['point', 'multipoint'].forEach(geometryType => {[false, true].forEach(gpu => {test(`#PointLayer -> batched Arrow ${geometryType} accessors (${diff --git a/test/node/processors/file-handler-test.js b/test/node/processors/file-handler-test.jsindex 5068bcff8..37ccb463d 100644--- a/test/node/processors/file-handler-test.js+++ b/test/node/processors/file-handler-test.js@@ -25,6 +25,8 @@ import {getDatasetRefreshIntervalMs} from '@kepler.gl/constants';import * as arrow from 'apache-arrow';+// The standalone bundle has distinct constructors, like DuckDB's bundled Arrow.+import * as foreignArrow from 'apache-arrow/Arrow.es2015.min';import {convertArrowToSchema} from '@loaders.gl/schema-utils';import {parsedFields, parsedRows} from 'test/fixtures/row-object';import {@@ -515,6 +517,135 @@ test('#file-handler -> parseGeoArrowBatchesOnWorkers parses the latest chunk', at.end();});+test('#file-handler -> separated points reach workers and fallback with all coordinates', async t => {+ const type = new arrow.Struct(+ ['x', 'y', 'z'].map(name => new arrow.Field(name, new arrow.Float64(), false))+ );+ const batches = [+ [+ {x: 10, y: 20, z: 5},+ {x: 30, y: 40, z: 15}+ ],+ [{x: 50, y: 60, z: 25}]+ ].map(points => {+ const table = new arrow.Table({geometry: arrow.vectorFromArray(points, type)});+ table.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT);+ return table;+ });+ const table = arrow.tableFromIPC(arrow.tableToIPC(batches[0].concat(batches[1]), 'file'));+ const bytes = arrow.tableToIPC(table, 'file');+ const processed = processArrowBatches(table.batches, {compact: false});+ for (const failWorker of [false, true]) {+ const result = await parseGeoArrowBatchesOnWorkers(processed, {+ parseOnWorker: async input => {+ t.equal(input.chunkData.type.listSize, 3, 'worker input has XYZ coordinate size');+ if (failWorker) throw new Error('worker unavailable');+ return parseGeoArrowWorkerInput(input);+ }+ });+ const {layer} = await layerFromFile({+ info: {id: 'separated'},+ data: {...processed, geoArrowBinaryData: result}+ });+ t.deepEqual(+ layer.dataToFeature.flatMap(data => Array.from(data.points.positions.value)),+ [10, 20, 5, 30, 40, 15, 50, 60, 25],+ 'worker/cache and fallback retain all axes'+ );+ t.deepEqual(+ layer.meta.bounds,+ [10, 20, 50, 60],+ 'cached bounds include latitude from every batch'+ );+ t.deepEqual(+ layer.dataToFeature.flatMap(data => Array.from(data.points.globalFeatureIds.value)),+ [0, 1, 2],+ 'cached feature ids retain unequal batch offsets'+ );+ }+ t.deepEqual(arrow.tableToIPC(table, 'file'), bytes, 'worker preparation preserves source data');+ t.end();+});++test('#file-handler -> worker fallback accepts points and lines from another Arrow runtime', async t => {+ const pointType = new foreignArrow.FixedSizeList(+ 2,+ new foreignArrow.Field('xy', new foreignArrow.Float64())+ );+ for (const encoding of [GEOARROW_EXTENSIONS.POINT, GEOARROW_EXTENSIONS.LINESTRING]) {+ const isPoint = encoding === GEOARROW_EXTENSIONS.POINT;+ const chunks = [+ [+ [10, 20],+ [30, 40]+ ],+ [[50, 60]]+ ].map(points => {+ // Exact child/offset lengths keep unrelated IPC padding out of line fixtures.+ const coordinates = isPoint ? points : points.flatMap(point => [point, point]);+ const child = foreignArrow+ .vectorFromArray(coordinates, pointType)+ .data[0].slice(0, coordinates.length);+ return isPoint+ ? child+ : foreignArrow.makeData({+ type: new foreignArrow.List(new foreignArrow.Field('vertices', pointType)),+ length: points.length,+ valueOffsets: Int32Array.from(points.map((_, i) => i * 2).concat(points.length * 2)),+ child+ });+ });+ const geoColumn = foreignArrow.makeVector(chunks);+ const field = {+ name: 'geometry',+ type: 'geoarrow',+ fieldIdx: 0,+ metadata: new Map([[GEOARROW_METADATA_KEY, encoding]])+ };+ const table = new foreignArrow.Table({geometry: geoColumn});+ const originalBytes = foreignArrow.tableToIPC(table);+ t.notOk(chunks[0] instanceof arrow.Data, 'fixture uses a separate Arrow runtime');++ for (const latestOnly of [false, true]) {+ const result = await parseGeoArrowBatchesOnWorkers(+ {fields: [field], cols: [geoColumn]},+ {+ latestOnly,+ parseOnWorker: async () => {+ throw new Error('worker unavailable');+ }+ }+ );+ t.deepEqual(Object.keys(result[0]), latestOnly ? ['1'] : ['0', '1'], 'selected batches');+ for (const [chunkIndex, binary] of Object.entries(result[0])) {+ const last = chunkIndex === '1';+ const points = last+ ? [[50, 60]]+ : [+ [10, 20],+ [30, 40]+ ];+ const rowIds = last ? [2] : [0, 1];+ const geometry = binary.binaryGeometries[0][isPoint ? 'points' : 'lines'];+ t.deepEqual(+ Array.from(geometry.positions.value),+ (isPoint ? points : points.flatMap(point => [point, point])).flat(),+ 'fallback preserves all coordinates'+ );+ t.deepEqual(+ Array.from(geometry.globalFeatureIds.value),+ isPoint ? rowIds : rowIds.flatMap(id => [id, id]),+ 'fallback preserves global row ids'+ );+ t.deepEqual(binary.bounds, last ? [50, 60, 50, 60] : [10, 20, 30, 40], 'bounds');+ t.deepEqual(binary.meanCenters, points, 'centroids remain aligned with rows');+ }+ }+ t.deepEqual(foreignArrow.tableToIPC(table), originalBytes, 'fallback preserves source data');+ }+ t.end();+});+test('#file-handler -> parseGeoArrowBatchesOnWorkers combines compacted worker chunks', async t => {const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false));const batchA = new arrow.Table({diff --git a/test/node/utils/layer-utils-test.js b/test/node/utils/layer-utils-test.jsindex 09955e878..5cf571f4f 100644--- a/test/node/utils/layer-utils-test.js+++ b/test/node/utils/layer-utils-test.js@@ -30,6 +30,10 @@ import {findDefaultColorFieldWithValues} from '@kepler.gl/utils';import * as arrow from 'apache-arrow';+// The standalone bundle has distinct constructors, like DuckDB's bundled Arrow.+import * as foreignArrow from 'apache-arrow/Arrow.es2015.min';+import {convertStructToFixedSizeList} from '../../../src/deckgl-arrow-layers/src/utils/utils';+import {GeoArrowScatterplotLayer} from '@kepler.gl/deckgl-arrow-layers';const defaultColorLayerTypes = ['a5', 's2', 'geohash', 'hexagonId', 'point', 'icon'];@@ -278,6 +282,223 @@ test('layerUtils -> createGeoArrowPointVector handles empty and nonempty coordint.end();});+test('layerUtils -> separated point normalization preserves validity, slices and precision', t => {+ for (const FloatType of [arrow.Float32, arrow.Float64]) {+ for (const dimensions of [2, 3]) {+ const names = ['x', 'y', 'z'].slice(0, dimensions);+ const type = new arrow.Struct(names.map(name => new arrow.Field(name, new FloatType())));+ const vector = arrow+ .vectorFromArray(+ [+ {x: -100, y: -80, z: -1},+ null,+ {x: 10.123456789, y: 20.123456789, z: 5.123456789},+ {x: NaN, y: NaN, z: NaN},+ {x: Infinity, y: 40, z: 1},+ {x: 50, y: null, z: 1},+ {x: 30, y: 40, z: 15}+ ],+ type+ )+ .slice(1);+ const data = vector.data[0];+ const normalized = convertStructToFixedSizeList(data);+ const validPoint = names.map(name => vector.get(1)[name]);+ const expected = [+ ...Array(dimensions).fill(NaN),+ ...validPoint,+ ...Array(3 * dimensions).fill(NaN),+ ...names.map(name => vector.get(5)[name])+ ];+ t.deepEqual(+ Array.from(normalized.children[0].values),+ expected,+ 'all axes retain source precision; missing and nonfinite rows are NaN'+ );+ t.equal(arrow.makeVector(normalized).get(0), null, 'slice retains parent null validity');+ t.deepEqual(+ Array.from(arrow.makeVector(normalized).get(1)),+ validPoint,+ 'slice retains valid row values'+ );+ t.ok(+ normalized.children[0].values instanceof Float64Array,+ 'does not narrow coordinate precision'+ );++ const metadata = new Map([[GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT]]);+ const geoField = {name: 'geometry', type: 'geoarrow', fieldIdx: 0, metadata};+ const dataContainer = new ArrowDataContainer({cols: [vector], fields: [geoField]});+ const meta = getGeojsonLayerMetaFromArrow({+ dataContainer,+ geoColumn: vector,+ geoField,+ columnIndex: 0+ });+ t.deepEqual(+ Array.from(meta.dataToFeature[0].points.positions.value),+ expected,+ 'GeoJSON uses normalized rows'+ );+ t.deepEqual(+ meta.bounds,+ [validPoint[0], validPoint[1], 30, 40],+ 'null and nonfinite rows do not distort bounds'+ );+ const table = new arrow.Table({geometry: vector});+ table.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT);+ const [layer] = new GeoArrowScatterplotLayer({data: table}).renderLayers();+ t.deepEqual(+ Array.from(layer.props.data.attributes.getPosition.value),+ expected,+ 'Point metadata inference uses normalized rows'+ );+ t.equal(layer.props.data.length, vector.length, 'invalid rows retain picking alignment');+ }+ }+ const type = new arrow.Struct(['x', 'y'].map(name => new arrow.Field(name, new arrow.Float64())));+ for (const rows of [[], [null, null]]) {+ const input = arrow.vectorFromArray(rows, type).data[0];+ const output = convertStructToFixedSizeList(input);+ t.equal(output.length, rows.length, 'empty/all-null input keeps its row count');+ t.equal(output.type.listSize, 2, 'empty/all-null coordinates still have two dimensions');+ t.ok(+ Array.from(output.children[0].values).every(Number.isNaN),+ 'missing points never become the origin'+ );+ }+ for (const FloatType of [arrow.Float32, arrow.Float64]) {+ const type = new arrow.FixedSizeList(2, new arrow.Field('xy', new FloatType()));+ const input = arrow+ .vectorFromArray(+ [+ [0, 0],+ [10, 20],+ [30, 40]+ ],+ type+ )+ .slice(1).data[0];+ t.equal(+ convertStructToFixedSizeList(input),+ input,+ 'interleaved slices retain the zero-copy fast path'+ );+ }+ t.end();+});++test('layerUtils -> incremental separated points retain unequal batch row offsets', t => {+ const type = new arrow.Struct(['x', 'y'].map(name => new arrow.Field(name, new arrow.Float64())));+ const chunks = [+ [+ {x: 10, y: 20},+ {x: 30, y: 40}+ ],+ [{x: 50, y: 60}],+ [{x: 70, y: 80}]+ ].map(rows => arrow.vectorFromArray(rows, type).data[0]);+ const geoColumn = arrow.makeVector(chunks);+ const metadata = new Map([[GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT]]);+ const geoField = {name: 'geometry', type: 'geoarrow', fieldIdx: 0, metadata};+ const dataContainer = new ArrowDataContainer({cols: [geoColumn], fields: [geoField]});+ const result = getGeojsonLayerMetaFromArrow({+ dataContainer,+ geoColumn,+ geoField,+ columnIndex: 0,+ chunkIndex: 2+ });+ t.equal(result.dataToFeature.length, 1, 'only converts the requested batch');+ t.deepEqual(+ Array.from(result.dataToFeature[0].points.positions.value),+ [70, 80],+ 'retains the requested coordinates'+ );+ t.deepEqual(+ Array.from(result.dataToFeature[0].points.globalFeatureIds.value),+ [3],+ 'sums preceding unequal batch lengths'+ );+ t.deepEqual(result.centroids, [[70, 80]], 'centroid uses both coordinate axes');+ t.end();+});++test('layerUtils -> native points from another Arrow runtime preserve slices and batch ids', t => {+ const type = new foreignArrow.FixedSizeList(+ 3,+ new foreignArrow.Field('xyz', new foreignArrow.Float64())+ );+ const first = foreignArrow+ .vectorFromArray(+ [+ [-100, -80, -1],+ [10, 20, 5],+ [30, 40, 15]+ ],+ type+ )+ .slice(1);+ const last = foreignArrow.vectorFromArray([[50, 60, 25]], type);+ const geoColumn = foreignArrow.makeVector([...first.data, ...last.data]);+ const geoField = {+ name: 'geometry',+ type: 'geoarrow',+ fieldIdx: 0,+ metadata: new Map([[GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT]])+ };+ const table = new foreignArrow.Table({geometry: geoColumn});+ const originalBytes = foreignArrow.tableToIPC(table);+ const dataContainer = new ArrowDataContainer({+ cols: [geoColumn],+ fields: [geoField],+ arrowTable: table+ });+ t.notOk(geoColumn.data[0] instanceof arrow.Data, 'fixture uses a separate Arrow runtime');++ for (const chunkIndex of [undefined, 1]) {+ const result = getGeojsonLayerMetaFromArrow({+ dataContainer,+ geoColumn,+ geoField,+ columnIndex: 0,+ chunkIndex+ });+ const incremental = chunkIndex === 1;+ t.deepEqual(+ result.dataToFeature.flatMap(data => Array.from(data.points.positions.value)),+ incremental ? [50, 60, 25] : [10, 20, 5, 30, 40, 15, 50, 60, 25],+ 'preserves XYZ coordinates and excludes padding and sliced-out rows'+ );+ t.deepEqual(+ result.dataToFeature.flatMap(data => Array.from(data.points.globalFeatureIds.value)),+ incremental ? [2] : [0, 1, 2],+ 'retains row ids across unequal batches'+ );+ t.deepEqual(result.bounds, incremental ? [50, 60, 50, 60] : [10, 20, 50, 60], 'bounds');+ t.deepEqual(+ result.centroids,+ incremental+ ? [[50, 60]]+ : [+ [10, 20],+ [30, 40],+ [50, 60]+ ],+ 'centroids remain aligned with rows'+ );+ result.dataToFeature.forEach((data, i) => {+ t.equal(+ data.points.positions.value.buffer,+ geoColumn.data[(chunkIndex || 0) + i].children[0].values.buffer,+ 'interleaved coordinates still share their source buffer'+ );+ });+ }+ t.deepEqual(foreignArrow.tableToIPC(table), originalBytes, 'does not change the source table');+ t.end();+});+test('layerUtils -> getGeojsonLayerMetaFromArrow uses worker-prepared data', t => {const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false));const geoColumn = arrow.vectorFromArray([[-122.4, 37.8]], pointType);