Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix: collapse Arrow record batches above configurable cap (#3663)
Igor Dykhtad810e51cf5cb
diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.tsindex f02db4eb2..4d5c38a0c 100644--- a/src/duckdb/src/table/duckdb-table.ts+++ b/src/duckdb/src/table/duckdb-table.ts@@ -23,6 +23,7 @@ import {getApplicationConfig,DatabaseAdapter,DatabaseConnection,+ compactArrowTable,isArrowTable,isArrowVector} from '@kepler.gl/utils';@@ -257,11 +258,13 @@ export class KeplerGlDuckDbTable extends KeplerTable {restoreGeoarrowMetadata(arrowResult, geoarrowMetadata);+ const compactedResult = compactArrowTable(arrowResult);+fields = useNewFields- ? arrowSchemaToFields(arrowResult, tableDuckDBTypes)- : data.fields ?? arrowSchemaToFields(arrowResult, tableDuckDBTypes);- cols = [...Array(arrowResult.numCols).keys()]- .map(i => arrowResult.getChildAt(i))+ ? arrowSchemaToFields(compactedResult, tableDuckDBTypes)+ : data.fields ?? arrowSchemaToFields(compactedResult, tableDuckDBTypes);+ cols = [...Array(compactedResult.numCols).keys()]+ .map(i => compactedResult.getChildAt(i)).filter(col => col) as arrow.Vector[];} catch (error) {console.error('DuckDB table: createTableAndGetArrow', error);diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.tsindex eb72f96bf..c5f9c4705 100644--- a/src/layers/src/geojson-layer/geojson-layer.ts+++ b/src/layers/src/geojson-layer/geojson-layer.ts@@ -617,9 +617,33 @@ export default class GeoJsonLayer extends Layer {const geoField = geoFieldAccessor(this.config.columns)(dataContainer);// update the latest batch/chunk of geoarrow data when loading data incrementally- if (geoColumn && geoField && this.dataToFeature.length < dataContainer.numChunks()) {- // for incrementally loading data, we only load and render the latest batch; otherwise, we will load and render all batches- const isIncrementalLoad = dataContainer.numChunks() - this.dataToFeature.length === 1;+ const processedRows = this.centroids.length;+ const numRows = dataContainer.numRows();+ const numChunks = dataContainer.numChunks();+ // WKB (and other whole-column collections) keep dataToFeature.length at 1+ // even when numChunks() is large. Row count is the progress signal then;+ // comparing length to numChunks would re-parse WKB on every meta update.+ const processedWholeColumn =+ this.dataToFeature.length > 0 && processedRows > 0 && processedRows >= numRows;+ // Progressive loads keep one dataToFeature entry per chunk, then compact+ // the finished table to one chunk. Row count already matches, so without+ // this check we would skip the rebuild and keep excess Deck layers.+ const compactedAfterProgressiveLoad = this.dataToFeature.length > numChunks;+ const needsUpdate =+ this.dataToFeature.length === 0 ||+ (!processedWholeColumn && this.dataToFeature.length < numChunks) ||+ (processedRows > 0 && processedRows < numRows) ||+ compactedAfterProgressiveLoad;++ if (geoColumn && geoField && needsUpdate) {+ // Incremental only when a new chunk appeared. Compacted tables stay at 1+ // chunk while row count grows, so those updates reprocess the whole table.+ const isIncrementalLoad =+ this.dataToFeature.length > 0 &&+ !processedWholeColumn &&+ numChunks - this.dataToFeature.length === 1 &&+ processedRows > 0 &&+ processedRows < numRows;// TODO: add support for COLUMN_MODE_TABLE in getGeojsonLayerMetaFromArrowconst {dataToFeature, bounds, fixedRadius, featureTypes, centroids} =getGeojsonLayerMetaFromArrow({@@ -628,9 +652,14 @@ export default class GeoJsonLayer extends Layer {geoField,...(isIncrementalLoad ? {chunkIndex: this.dataToFeature.length} : null)});- if (centroids) this.centroids = this.centroids.concat(centroids);+ if (isIncrementalLoad) {+ if (centroids) this.centroids = this.centroids.concat(centroids);+ this.dataToFeature = [...this.dataToFeature, ...dataToFeature];+ } else {+ this.centroids = centroids || [];+ this.dataToFeature = dataToFeature;+ }this.updateMeta({bounds, fixedRadius, featureTypes});- this.dataToFeature = [...this.dataToFeature, ...dataToFeature];}} else if (this.dataToFeature.length === 0 || this.config.columnMode === COLUMN_MODE_TABLE) {const getFeature = this.getPositionAccessor(dataContainer);diff --git a/src/layers/src/heatmap-layer/heatmap-layer.ts b/src/layers/src/heatmap-layer/heatmap-layer.tsindex eabf1289c..f5685ae74 100644--- a/src/layers/src/heatmap-layer/heatmap-layer.ts+++ b/src/layers/src/heatmap-layer/heatmap-layer.ts@@ -30,7 +30,7 @@ import {booleanWithin} from '@turf/boolean-within';import {point as turfPoint} from '@turf/helpers';import {Feature, Polygon} from 'geojson';-import {getGeoArrowPointLayerProps, FindDefaultLayerPropsReturnValue} from '../layer-utils';+import {getGeoArrowPointLayerProps, FindDefaultLayerPropsReturnValue, getGeoArrowPointCoords} from '../layer-utils';import {getFilterDataFunc} from '../aggregation-layer';import {parseGeoJsonRawFeature,@@ -88,7 +88,8 @@ export const geoarrowPosAccessor =(dc: DataContainerInterface) =>(d: {index: number}): number[] => {const row = dc.valueAt(d.index, geoarrow.fieldIdx);- return [row.get(0), row.get(1)];+ const coords = getGeoArrowPointCoords(row);+ return [coords[0], coords[1]];};export const geojsonAccessor =diff --git a/src/layers/src/layer-utils.ts b/src/layers/src/layer-utils.tsindex 72ab21bca..ee074093e 100644--- a/src/layers/src/layer-utils.ts+++ b/src/layers/src/layer-utils.ts@@ -24,14 +24,17 @@ import {convertGeoArrowGeometryToGeoJSON} from '@loaders.gl/gis';import {WKBLoader} from '@loaders.gl/wkt';import {geojsonToBinary} from '@loaders.gl/gis';import {- BinaryFeatureCollection,Geometry,BinaryPointFeature,BinaryLineFeature,BinaryPolygonFeature} from '@loaders.gl/schema';-import {DeckGlGeoTypes, GeojsonDataMaps} from './geojson-layer/geojson-utils';+import {+ DeckGlGeoTypes,+ GeojsonDataMaps,+ getCentroidFromGeometry+} from './geojson-layer/geojson-utils';export type FindDefaultLayerProps = {label: string;@@ -87,7 +90,7 @@ function getBinaryGeometriesFromWKBArrow(geoColumn: arrow.Vector,options: {chunkIndex?: number; chunkOffset?: number}): GeojsonLayerMetaProps {- const dataToFeature: BinaryFeatureCollection[] = [];+ const centroids: Array<number[] | null> = [];const featureTypes: GeojsonLayerMetaProps['featureTypes'] = {point: false,line: false,@@ -102,8 +105,8 @@ function getBinaryGeometriesFromWKBArrow(let featureIndex = globalFeatureIdOffset;let bounds: [number, number, number, number] = [Infinity, Infinity, -Infinity, -Infinity];+ const geojsonFeatures: Feature[] = [];chunks.forEach(chunk => {- const geojsonFeatures: Feature[] = [];for (let i = 0; i < chunk.length; ++i) {// ignore features without any geometryif (chunk.valueOffsets[i + 1] - chunk.valueOffsets[i] > 0) {@@ -118,54 +121,57 @@ function getBinaryGeometriesFromWKBArrow(properties: {index: featureIndex}};geojsonFeatures.push(feature);+ centroids.push(getCentroidFromGeometry(geometry));const {type} = geometry;featureTypes.polygon = type === 'Polygon' || type === 'MultiPolygon';featureTypes.point = type === 'Point' || type === 'MultiPoint';featureTypes.line = type === 'LineString' || type === 'MultiLineString';+ } else {+ centroids.push(null);}featureIndex++;}+ });- const geojsonToBinaryOptions = {- triangulate: true,- fixRingWinding: true- };- const binaryFeatures = geojsonToBinary(geojsonFeatures, geojsonToBinaryOptions);-- // Need to update globalFeatureIds, to take into account previous batches,- // as geojsonToBinary doesn't have such option.- const featureTypesArr = ['points', 'lines', 'polygons'];- featureTypesArr.forEach(prop => {- const features = binaryFeatures[prop] as- | BinaryPointFeature- | BinaryLineFeature- | BinaryPolygonFeature;- if (features) {- bounds = updateBoundsFromGeoArrowSamples(- features.positions.value as Float64Array,- features.positions.size,- bounds- );-- const {globalFeatureIds, numericProps} = features;- const {index} = numericProps;- const len = globalFeatureIds.value.length;- for (let i = 0; i < len; ++i) {- globalFeatureIds.value[i] = index.value[i];- }+ // One BinaryFeatureCollection for the whole column. A collection per Arrow+ // chunk becomes one GeoJsonLayer (with fill/stroke/point sublayers) each, and+ // parquet/DuckDB files easily exceed deck.gl's 255 pickable-layer cap.+ const geojsonToBinaryOptions = {+ triangulate: true,+ fixRingWinding: true+ };+ const binaryFeatures = geojsonToBinary(geojsonFeatures, geojsonToBinaryOptions);++ const featureTypesArr = ['points', 'lines', 'polygons'];+ featureTypesArr.forEach(prop => {+ const features = binaryFeatures[prop] as+ | BinaryPointFeature+ | BinaryLineFeature+ | BinaryPolygonFeature;+ if (features) {+ bounds = updateBoundsFromGeoArrowSamples(+ features.positions.value as Float64Array,+ features.positions.size,+ bounds+ );++ const {globalFeatureIds, numericProps} = features;+ const {index} = numericProps;+ const len = globalFeatureIds.value.length;+ for (let i = 0; i < len; ++i) {+ globalFeatureIds.value[i] = index.value[i];}- });-- dataToFeature.push(binaryFeatures);+ }});return {- dataToFeature: dataToFeature,+ dataToFeature: [binaryFeatures],featureTypes: featureTypes,bounds,- fixedRadius: false+ fixedRadius: false,+ centroids};}@@ -290,6 +296,38 @@ export function getGeoArrowPointFields(fields: Field[]): Field[] {});}+/**+ * Reads lng/lat from a GeoArrow point cell. Arrow may return a FixedSizeList+ * (with .get), a typed array, a plain [lng, lat] array, or a {x,y} struct.+ */+export function getGeoArrowPointCoords(row: any): [number, number, number] {+ if (row == null) {+ return [NaN, NaN, 0];+ }+ if (typeof row.get === 'function') {+ const x = row.get(0) ?? row.get('x') ?? row.get('lng') ?? row.get('lon');+ const y = row.get(1) ?? row.get('y') ?? row.get('lat');+ if (Number.isFinite(x) && Number.isFinite(y)) {+ return [Number(x), Number(y), 0];+ }+ }+ if (typeof row.toArray === 'function') {+ const arr = row.toArray();+ return [Number(arr[0]), Number(arr[1]), 0];+ }+ if (Array.isArray(row) || ArrayBuffer.isView(row)) {+ return [Number(row[0]), Number(row[1]), 0];+ }+ if (typeof row === 'object') {+ const x = row.x ?? row.lng ?? row.lon;+ const y = row.y ?? row.lat;+ if (x !== undefined && y !== undefined) {+ return [Number(x), Number(y), 0];+ }+ }+ return [NaN, NaN, 0];+}+/*** Builds an arrow vector compatible with ARROW:extension:name geoarrow.point.* @param getPosition Position accessor.@@ -305,7 +343,6 @@ export function createGeoArrowPointVector(// in a correct arrow format, as this approach seems too excessive for just a simple interleaved buffer.const numElements = dataContainer.numRows();- const table = dataContainer.getTable();const numCoords = numElements > 0 ? getPosition({index: 0}).length : 2;const precision = 2;@@ -319,21 +356,16 @@ export function createGeoArrowPointVector(const fixedSizeListBuilder = new arrow.FixedSizeListBuilder({type: fixedSizeList});fixedSizeListBuilder.addChild(floatBuilder);- const assembledBatches: arrow.Data[] = [];+ // One record batch, not one per source batch: a batch per parquet/DuckDB+ // chunk would create hundreds of pickable ScatterplotLayers and hit deck.gl's+ // 255-layer picking cap.const indexData = {index: 0};- for (let batchIndex = 0; batchIndex < table.batches.length; ++batchIndex) {- const numRowsInBatch = table.batches[batchIndex].numRows;-- for (let i = 0; i < numRowsInBatch; ++i) {- const pos = getPosition(indexData);- fixedSizeListBuilder.append(pos);-- ++indexData.index;- }- assembledBatches.push(fixedSizeListBuilder.flush());+ for (let i = 0; i < numElements; ++i) {+ indexData.index = i;+ fixedSizeListBuilder.append(getPosition(indexData));}- return arrow.makeVector(assembledBatches);+ return arrow.makeVector(numElements > 0 ? [fixedSizeListBuilder.flush()] : []);}/**diff --git a/src/layers/src/point-layer/point-layer.ts b/src/layers/src/point-layer/point-layer.tsindex 5ab058bf9..5045cca98 100644--- a/src/layers/src/point-layer/point-layer.ts+++ b/src/layers/src/point-layer/point-layer.ts@@ -41,7 +41,8 @@ import {createGeoArrowPointVector,getFilteredIndex,getNeighbors,- FindDefaultLayerProps+ FindDefaultLayerProps,+ getGeoArrowPointCoords} from '../layer-utils';import {getGeojsonPointDataMaps, GeojsonPointDataMaps} from '../geojson-layer/geojson-utils';import {@@ -131,7 +132,7 @@ export const geoarrowPosAccessor =(dataContainer: DataContainerInterface) =>(d: {index: number}) => {const row = dataContainer.valueAt(d.index, geoarrow.fieldIdx);- return [row.get(0), row.get(1), 0];+ return getGeoArrowPointCoords(row);};export const COLUMN_MODE_POINTS = 'points';diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.tsindex 2759426b4..17b9c011f 100644--- a/src/processors/src/data-processor.ts+++ b/src/processors/src/data-processor.ts@@ -20,6 +20,7 @@ import {ProcessorResult, Field} from '@kepler.gl/types';import {arrowDataTypeToAnalyzerDataType,arrowDataTypeToFieldType,+ compactArrowTable,hasOwnProperty,isPlainObject} from '@kepler.gl/utils';@@ -616,17 +617,34 @@ function castBigIntColumnsToFloat64(arrowTable: arrow.Table): arrow.Table {return new arrow.Table(newColumns);}+export type ProcessArrowBatchesOptions = {+ /**+ * When false, skip compactArrowTable. Progressive Arrow loading uses this so+ * each incoming batch does not copy the accumulated table. Defaults to true.+ * The final processFileContent call leaves this unset so the completed table+ * is compacted once.+ */+ compact?: boolean;+};+/*** Parse arrow batches returned from parseInBatches()*- * @param arrowTable the arrow table to parse+ * @param arrowBatches the arrow record batches to parse+ * @param options optional processing flags* @returns dataset containing `fields` and `rows` or null*/-export function processArrowBatches(arrowBatches: arrow.RecordBatch[]): ProcessorResult | null {+export function processArrowBatches(+ arrowBatches: arrow.RecordBatch[],+ options?: ProcessArrowBatchesOptions+): ProcessorResult | null {if (arrowBatches.length === 0) {return null;}- const arrowTable = castBigIntColumnsToFloat64(new arrow.Table(arrowBatches));+ const arrowTable =+ options?.compact === false+ ? castBigIntColumnsToFloat64(new arrow.Table(arrowBatches))+ : compactArrowTable(castBigIntColumnsToFloat64(new arrow.Table(arrowBatches)));const fields = arrowSchemaToFields(arrowTable);const cols = [...Array(arrowTable.numCols).keys()].map(i => arrowTable.getChildAt(i));diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.tsindex 2e229c93b..8d090cb26 100644--- a/src/processors/src/file-handler.ts+++ b/src/processors/src/file-handler.ts@@ -70,6 +70,11 @@ export type ProcessFileDataContent = {sourceUrl?: string;/** User-selected or inferred remote file format (csv, geojson, parquet, …). */keplerFormat?: string;+ /**+ * When true, skip Arrow record-batch compaction. Set on intermediate+ * progressive Arrow loads so each batch does not copy the growing table.+ */+ skipArrowCompact?: boolean;};export {isArrowTable};@@ -279,7 +284,10 @@ export async function processFileData({// eslint-disable-next-line no-useless-catchlet result;try {- result = await processor(data);+ result =+ format === DATASET_FORMATS.arrow && content.skipArrowCompact+ ? await processArrowBatches(data as any, {compact: false})+ : await processor(data);} catch (error) {throw new Error(`Can not process uploaded file, ${getError(error as Error)}`);}diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.tsindex d037d9186..cfa4e5ca6 100644--- a/src/reducers/src/vis-state-updaters.ts+++ b/src/reducers/src/vis-state-updaters.ts@@ -3517,7 +3517,12 @@ export const nextFileBatchUpdater = (fileName.endsWith('arrow') &&accumulated?.data?.length > 0? [- PROCESS_FILE_DATA({content: accumulated, fileCache: []}).bimap(+ // Skip compaction while batches are still arriving. The completed+ // file is compacted once in processFileContent.+ PROCESS_FILE_DATA({+ content: {...accumulated, skipArrowCompact: true},+ fileCache: []+ }).bimap(result => loadFilesSuccess(result),err => loadFilesErr(fileName, err))diff --git a/src/utils/src/application-config.ts b/src/utils/src/application-config.tsindex 0245f4057..d5cd5b05e 100644--- a/src/utils/src/application-config.ts+++ b/src/utils/src/application-config.ts@@ -224,6 +224,22 @@ export type KeplerApplicationConfig = {/** Controls Redux action logging verbosity in dev mode. See {@link ReduxLogLevel}. */reduxLogLevel?: ReduxLogLevel;++ /**+ * If an Arrow/Parquet table has more record batches than this, they are+ * compacted into a single batch. The default is `1`, so any table with more+ * than one record batch is compacted. Deck.gl picking supports at most 255+ * pickable leaf layers, and GeoJSON adds fill/stroke/point sublayers per+ * batch, so a higher cap still overruns picking. Compacting copies the table.+ * Progressive Arrow loading skips compaction on intermediate batches and+ * compacts the completed file once. Set a very large number to never compact.+ *+ * @example+ * ```+ * initApplicationConfig({maxArrowBatches: 64});+ * ```+ */+ maxArrowBatches?: number;};const DEFAULT_APPLICATION_CONFIG: Required<KeplerApplicationConfig> = {@@ -319,7 +335,9 @@ const DEFAULT_APPLICATION_CONFIG: Required<KeplerApplicationConfig> = {customIconUrl: '',- reduxLogLevel: 1+ reduxLogLevel: 1,++ maxArrowBatches: 1};const applicationConfig: Required<KeplerApplicationConfig> = DEFAULT_APPLICATION_CONFIG;diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.tsindex f462b11db..698bf0e18 100644--- a/src/utils/src/arrow-data-container.ts+++ b/src/utils/src/arrow-data-container.ts@@ -7,6 +7,7 @@ import * as arrow from 'apache-arrow';import {console as globalConsole} from 'global/window';import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer';+import {getApplicationConfig} from './application-config';import {DataContainerInterface, RangeOptions} from './data-container-interface';import {DataRow, SharedRowOptions} from './data-row';@@ -120,6 +121,185 @@ function* columnIterator(dataContainer: DataContainerInterface, columnIndex: num}}+const DEFAULT_MAX_ARROW_BATCHES = 1;++/**+ * Collapse Arrow record batches into as few chunks as possible.+ *+ * Deck.gl picking encodes at most 255 pickable leaf layers. Kepler creates one+ * Deck layer per Arrow record batch (and GeoJsonLayer adds fill/stroke/point+ * sublayers), so parquet/DuckDB tables with hundreds of batches make hover and+ * click miss most of the data. Combining chunks keeps picking under that cap+ * without changing row values.+ *+ * apache-arrow JS does not expose Vector.combineChunks in the version Kepler+ * uses, so we rebuild each column through a Builder (one chunk per column).+ *+ * Compaction only runs when `table.batches.length` is greater than+ * `maxArrowBatches` from {@link getApplicationConfig} (default `1`) or the+ * optional override. Progressive Arrow loading does not compact on each+ * incoming batch; {@link ArrowDataContainer.update} assigns new chunks as-is+ * and the completed file is compacted once.+ */+export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number): arrow.Table {+ const batchLimit =+ maxArrowBatches ?? getApplicationConfig().maxArrowBatches ?? DEFAULT_MAX_ARROW_BATCHES;++ if (!table || !Array.isArray(table.batches) || table.batches.length <= 1) {+ return table;+ }++ if (table.batches.length <= batchLimit) {+ return table;+ }++ try {+ const columns: Record<string, arrow.Vector> = {};+ const seenNames = new Map<string, number>();+ for (let i = 0; i < table.numCols; i++) {+ const field = table.schema.fields[i];+ const column = table.getChildAt(i);+ if (!column) {+ continue;+ }+ // Arrow schemas may repeat field names; object keys would drop earlier columns.+ const seen = seenNames.get(field.name) ?? 0;+ seenNames.set(field.name, seen + 1);+ const key = seen === 0 ? field.name : `${field.name}_${seen}`;+ columns[key] = compactArrowVector(column, column.type || field.type);+ }++ // Always build a kepler apache-arrow Table. Using `table.constructor` can+ // mix DuckDB's bundled Arrow Table with rebuilt vectors and drop schema+ // metadata (`geo`, ARROW:extension:name) that Kepler needs to create layers.+ const compacted = tableFromCompactedColumns(table, columns);++ if (compacted?.batches?.length && compacted.batches.length < table.batches.length) {+ return compacted;+ }+ } catch {+ // Keep the original table if this Arrow build cannot combine chunks.+ }++ return table;+}++function copyArrowSchemaMetadata(from?: arrow.Schema | null, to?: arrow.Schema | null): void {+ if (!from || !to) {+ return;+ }++ const fromMeta = from.metadata as Map<string, string> | undefined;+ const toMeta = to.metadata as Map<string, string> | undefined;+ if (fromMeta && toMeta && typeof fromMeta.forEach === 'function' && typeof toMeta.set === 'function') {+ fromMeta.forEach((value, key) => {+ toMeta.set(key, value);+ });+ }++ const fromByName = new Map((from.fields || []).map(field => [field.name, field]));+ (to.fields || []).forEach((toField, index) => {+ const fromField = from.fields?.[index] || fromByName.get(toField.name);+ const src = fromField?.metadata as Map<string, string> | undefined;+ const dst = toField?.metadata as Map<string, string> | undefined;+ if (!src || !dst || typeof src.forEach !== 'function' || typeof dst.set !== 'function') {+ return;+ }+ src.forEach((value, key) => {+ dst.set(key, value);+ });+ });+}++function tableFromCompactedColumns(+ source: arrow.Table,+ columns: Record<string, arrow.Vector>+): arrow.Table {+ let compacted: arrow.Table;+ try {+ compacted = new arrow.Table(source.schema, columns);+ } catch {+ compacted = new arrow.Table(columns);+ }+ copyArrowSchemaMetadata(source.schema, compacted.schema);+ return compacted;+}++/**+ * Convert an Arrow cell from Vector.get() into a value makeBuilder.append accepts.+ * Nested GeoArrow (FixedSizeList / List) cells are Vectors, not arrays.+ */+function arrowCellToBuilderValue(value: unknown): unknown {+ if (value == null) {+ return null;+ }+ if (ArrayBuffer.isView(value)) {+ return Array.from(value as any);+ }++ const nested = value as {+ toArray?: () => ArrayLike<unknown>;+ numChildren?: number;+ get?: (index: number) => unknown;+ length?: number;+ isValid?: (index: number) => boolean;+ };++ // Primitive numeric child of a FixedSizeList (lng/lat).+ if (typeof nested.toArray === 'function' && nested.numChildren === 0) {+ return Array.from(nested.toArray());+ }++ if (typeof nested.get === 'function' && typeof nested.length === 'number') {+ const out: unknown[] = [];+ for (let i = 0; i < nested.length; i++) {+ const valid = typeof nested.isValid === 'function' ? nested.isValid(i) : true;+ out.push(valid ? arrowCellToBuilderValue(nested.get(i)) : null);+ }+ return out;+ }++ return value;+}++function compactArrowVector(column: arrow.Vector, type: arrow.DataType = column.type): arrow.Vector {+ if (!column || !column.data || column.data.length <= 1) {+ return column;+ }++ const maybeCombine = (column as {combineChunks?: () => arrow.Vector}).combineChunks;+ if (typeof maybeCombine === 'function') {+ try {+ const combined = maybeCombine.call(column);+ if (combined?.data?.length <= 1) {+ return combined;+ }+ } catch {+ // fall through to Builder+ }+ }++ const builder = arrow.makeBuilder({type, nullValues: [null]});+ const length = column.length;+ for (let i = 0; i < length; i++) {+ const isValid = typeof (column as {isValid?: (index: number) => boolean}).isValid === 'function'+ ? (column as {isValid: (index: number) => boolean}).isValid(i)+ : true;+ // FixedSizeList/List .get() returns nested Vectors. Appending those+ // silently writes NaN (points) or throws (lines/polygons). Plain JS+ // arrays are what makeBuilder expects.+ builder.append(isValid ? arrowCellToBuilderValue(column.get(i)) : null);+ }+ const finished = builder.finish() as arrow.Vector | {toVector: () => arrow.Vector};+ if (finished instanceof arrow.Vector) {+ return finished;+ }+ if (typeof (finished as {toVector?: () => arrow.Vector}).toVector === 'function') {+ return (finished as {toVector: () => arrow.Vector}).toVector();+ }+ return column;+}+/*** A data container where all data is stored in raw Arrow table*/@@ -144,14 +324,16 @@ export class ArrowDataContainer implements DataContainerInterface {throw Error("ArrowDataContainer: columns object isn't an array");}- this._cols = data.cols;- this._numColumns = data.cols.length;- this._numRows = data.cols[0].length;this._fields = data.fields || [];- this._numChunks = data.cols[0].data.length;- // this._colData = data.cols.map(c => c.toArray());-- this._arrowTable = data.arrowTable || this._createTable();+ this._cols = data.cols;+ this._numColumns = this._cols.length;+ this._numRows = this._cols[0]?.length ?? 0;+ this._numChunks = this._cols[0]?.data?.length ?? 0;+ const table = compactArrowTable(data.arrowTable || this._createTable());+ this._arrowTable = table;+ if (table.numCols > 0) {+ this._assignTable(table);+ }}/**@@ -170,20 +352,36 @@ export class ArrowDataContainer implements DataContainerInterface {return this._arrowTable;}+ private _assignTable(table: arrow.Table) {+ this._arrowTable = table;+ this._cols = Array.from(+ {length: table.numCols},+ (_, i) => table.getChildAt(i) as arrow.Vector+ ).filter(col => col);+ this._numColumns = this._cols.length;+ this._numRows = this._cols[0]?.length ?? 0;+ this._numChunks = this._cols[0]?.data?.length ?? 0;+ }+update(updateData: arrow.Vector<any>[] | arrow.Table) {+ // Incremental Arrow loads append batches here. Compacting on every update+ // would copy the growing table (Builder over every cell). Skip collapse;+ // processArrowBatches compact once when the file is fully loaded.const isArrow = isArrowTable(updateData);if (isArrow) {- this._cols = Array.from(- {length: updateData.numCols},- (_, i) => updateData.getChildAt(i) as arrow.Vector- ).filter(col => col);+ this._assignTable(updateData);} else {this._cols = updateData;+ const table = this._createTable();+ if (table.numCols > 0) {+ this._assignTable(table);+ } else {+ this._numColumns = this._cols.length;+ this._numRows = this._cols[0]?.length ?? 0;+ this._numChunks = this._cols[0]?.data?.length ?? 0;+ this._arrowTable = table;+ }}- this._numColumns = this._cols?.length ?? 0;- this._numRows = this._cols?.[0]?.length ?? 0;- this._numChunks = this._cols?.[0]?.data?.length ?? 0;- this._arrowTable = isArrow ? updateData : this._createTable();// cache column data to make valueAt() faster// this._colData = this._cols.map(c => c.toArray());diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.tsindex 954031a1e..a3ab46ede 100644--- a/src/utils/src/filter-utils.ts+++ b/src/utils/src/filter-utils.ts@@ -376,6 +376,21 @@ export function getFilterProps(}}+function hasFiniteLngLat(pos: unknown): pos is number[] {+ if (pos == null || !(Array.isArray(pos) || ArrayBuffer.isView(pos))) {+ return false;+ }+ const values = pos as ArrayLike<unknown>;+ // Do not coerce with Number(): Number(null/''/false) is 0 and would pass.+ return (+ values.length >= 2 &&+ typeof values[0] === 'number' &&+ Number.isFinite(values[0]) &&+ typeof values[1] === 'number' &&+ Number.isFinite(values[1])+ );+}+export const getPolygonFilterFunctor = (layer, filter, dataContainer) => {const getPosition = layer.getPositionAccessor(dataContainer);@@ -388,22 +403,15 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => {if (!coordinates) return false;if (Array.isArray(coordinates[0])) {return (coordinates as number[][]).some(- coord =>- coord.length >= 2 &&- coord.every(Number.isFinite) &&- isInPolygon(coord, filter.value)+ coord => hasFiniteLngLat(coord) && isInPolygon(coord, filter.value));}- return (- coordinates.length >= 2 &&- coordinates.every(Number.isFinite) &&- isInPolygon(coordinates, filter.value)- );+ return hasFiniteLngLat(coordinates) && isInPolygon(coordinates, filter.value);};}return data => {const pos = getPosition(data);- return pos.every(Number.isFinite) && isInPolygon(pos, filter.value);+ return hasFiniteLngLat(pos) && isInPolygon(pos, filter.value);};case LAYER_TYPES.grid:case LAYER_TYPES.hexagon:@@ -418,14 +426,15 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => {}return data => {const pos = getPosition(data);- return pos.every(Number.isFinite) && isInPolygon(pos, filter.value);+ return hasFiniteLngLat(pos) && isInPolygon(pos, filter.value);};case LAYER_TYPES.arc:case LAYER_TYPES.line:return data => {const pos = getPosition(data);return (- pos.every(Number.isFinite) &&+ hasFiniteLngLat(pos) &&+ hasFiniteLngLat([pos[3], pos[4]]) &&[[pos[0], pos[1]],[pos[3], pos[4]]@@ -446,7 +455,7 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => {return false;}const pos = getCentroid({id});- return pos.every(Number.isFinite) && isInPolygon(pos, filter.value);+ return hasFiniteLngLat(pos) && isInPolygon(pos, filter.value);};case LAYER_TYPES.geojson:return data => {@@ -461,7 +470,7 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => {}return data => {const pos = getPosition(data);- return pos.every(Number.isFinite) && isInPolygon(pos, filter.value);+ return hasFiniteLngLat(pos) && isInPolygon(pos, filter.value);};default:return () => true;@@ -830,7 +839,9 @@ export function isInRange(val: any, domain: number[]): boolean {* @return {boolean}*/export function isInPolygon(point: number[], polygon: any): boolean {- return booleanWithin(turfPoint(point), polygon);+ // turfPoint requires a plain array; Arrow accessors can return typed arrays+ const lngLat = Array.isArray(point) ? point : [Number(point[0]), Number(point[1])];+ return booleanWithin(turfPoint(lngLat), polygon);}export function getTimeWidgetTitleFormatter(domain: [number, number]): string | null {if (!isValidTimeDomain(domain)) {diff --git a/src/utils/src/index.ts b/src/utils/src/index.tsindex b38a85654..f730968ee 100644--- a/src/utils/src/index.ts+++ b/src/utils/src/index.ts@@ -144,6 +144,7 @@ export * from './map-utils';export {ArrowDataContainer,+ compactArrowTable,arrowDataTypeToAnalyzerDataType,arrowDataTypeToFieldType,isArrowTable,diff --git a/test/node/processors/file-handler-test.js b/test/node/processors/file-handler-test.jsindex fdb6cd782..b190b92fb 100644--- a/test/node/processors/file-handler-test.js+++ b/test/node/processors/file-handler-test.js@@ -2,7 +2,8 @@// Copyright contributors to the kepler.gl projectimport test from 'tape';-import {isKeplerGlMap, makeProgressIterator, filesToDataPayload, processFileData} from '@kepler.gl/processors';+import {isKeplerGlMap, makeProgressIterator, filesToDataPayload, processFileData, processArrowBatches} from '@kepler.gl/processors';+import * as arrow from 'apache-arrow';import {parsedFields, parsedRows} from 'test/fixtures/row-object';import {savedStateV1InteractionCoordinate as keplerglMap,@@ -277,3 +278,59 @@ test('#file-handler -> processFileData remote ids do not collide on filename', at.end();});++test('#file-handler -> processArrowBatches skip compact for incremental loads', t => {+ const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]);+ const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]);+ const combined = batchA.concat(batchB);++ t.ok(combined.batches.length > 1, 'fixture should have multiple record batches');++ const skipped = processArrowBatches(combined.batches, {compact: false});+ t.equal(+ skipped.cols[0].data.length,+ combined.batches.length,+ 'compact:false should leave record batches unchanged'+ );++ const compacted = processArrowBatches(combined.batches);+ t.equal(compacted.cols[0].data.length, 1, 'default processArrowBatches should compact');+ t.equal(compacted.cols[0].length, 2, 'compacted table should keep all rows');++ t.end();+});++test('#file-handler -> processFileData skipArrowCompact', async t => {+ const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]);+ const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]);+ const combined = batchA.concat(batchB);++ const incremental = await processFileData({+ content: {+ fileName: 'points.arrow',+ data: combined.batches,+ skipArrowCompact: true+ },+ fileCache: []+ });+ t.equal(+ incremental[0].data.cols[0].data.length,+ combined.batches.length,+ 'progressive processFileData should not compact Arrow batches'+ );++ const finished = await processFileData({+ content: {+ fileName: 'points.arrow',+ data: combined.batches+ },+ fileCache: []+ });+ t.equal(+ finished[0].data.cols[0].data.length,+ 1,+ 'final processFileData should compact Arrow batches'+ );++ t.end();+});diff --git a/test/node/utils/data-container-test.js b/test/node/utils/data-container-test.jsindex 200be5c68..7095ef48e 100644--- a/test/node/utils/data-container-test.js+++ b/test/node/utils/data-container-test.js@@ -3,7 +3,9 @@import test from 'tape';-import {createDataContainer, createIndexedDataContainer} from '@kepler.gl/utils';+import {createDataContainer, createIndexedDataContainer, compactArrowTable, ArrowDataContainer} from '@kepler.gl/utils';+import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants';+import * as arrow from 'apache-arrow';const data = [[10, 20], // 0@@ -120,3 +122,321 @@ test('IndexedDataContainer', t => {t.end();});++test('ArrowDataContainer -> compactArrowTable collapses record batches over the limit', t => {+ const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]);+ const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]);+ const combined = batchA.concat(batchB);++ t.ok(combined.batches.length > 1, 'fixture should have multiple record batches');++ const belowLimit = compactArrowTable(combined, 255);+ t.equal(+ belowLimit.batches.length,+ combined.batches.length,+ 'compactArrowTable should leave tables at or under maxArrowBatches unchanged'+ );++ combined.schema.metadata.set(+ 'geo',+ JSON.stringify({columns: {lng: {encoding: 'WKB'}}})+ );+ combined.schema.fields[0].metadata.set('ARROW:extension:name', 'geoarrow.wkb');++ const compacted = compactArrowTable(combined, 1);+ t.equal(compacted.numRows, 2, 'compactArrowTable should keep all rows');+ t.equal(compacted.batches.length, 1, 'compactArrowTable should collapse into one batch');+ t.ok(+ compacted.schema.metadata.get('geo'),+ 'compactArrowTable should keep GeoParquet schema metadata used to create layers'+ );+ t.equal(+ compacted.schema.fields[0].metadata.get('ARROW:extension:name'),+ 'geoarrow.wkb',+ 'compactArrowTable should keep geoarrow field metadata'+ );++ const cols = Array.from({length: combined.numCols}, (_, i) => combined.getChildAt(i)).filter(+ Boolean+ );+ const arrowDc = new ArrowDataContainer({+ cols,+ fields: combined.schema.fields.map((field, fieldIdx) => ({+ name: field.name,+ fieldIdx+ })),+ arrowTable: compactArrowTable(combined, 1)+ });++ t.equal(arrowDc.numRows(), 2, 'ArrowDataContainer should keep all rows');+ t.equal(arrowDc.numChunks(), 1, 'ArrowDataContainer should store a compacted table');+ t.deepEqual(arrowDc.valueAt(0, 0), -122.4, 'compacted container should preserve values');++ t.end();+});++test('ArrowDataContainer -> update does not collapse record batches', t => {+ const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]);+ const colsA = Array.from({length: batchA.numCols}, (_, i) => batchA.getChildAt(i)).filter(Boolean);+ const dc = new ArrowDataContainer({+ cols: colsA,+ fields: batchA.schema.fields.map((field, fieldIdx) => ({+ name: field.name,+ fieldIdx+ })),+ arrowTable: batchA+ });++ const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]);+ const combined = batchA.concat(batchB);+ dc.update(combined);++ t.equal(dc.numRows(), 2, 'update should keep all rows');+ t.equal(+ dc.numChunks(),+ combined.batches.length,+ 'update should not compact batches during incremental loading'+ );+ t.deepEqual(dc.valueAt(1, 0), -122.5, 'update should preserve newly appended values');++ t.end();+});++function finishArrowVector(builder) {+ const finished = builder.finish();+ if (finished instanceof arrow.Vector) {+ return finished;+ }+ return finished.toVector();+}++function tableFromColumnValues(name, type, values, metadata) {+ const builder = arrow.makeBuilder({type, nullValues: [null]});+ values.forEach(value => builder.append(value));+ const vector = finishArrowVector(builder);+ const field = new arrow.Field(name, type, true, metadata);+ return new arrow.Table(new arrow.Schema([field]), {[name]: vector});+}++function concatColumnTables(name, type, leftValues, rightValues, metadata) {+ return tableFromColumnValues(name, type, leftValues, metadata).concat(+ tableFromColumnValues(name, type, rightValues, metadata)+ );+}++function jsArrowCell(value) {+ if (value == null) {+ return null;+ }+ if (ArrayBuffer.isView(value)) {+ return Array.from(value);+ }+ if (typeof value.toArray === 'function' && value.numChildren === 0) {+ return Array.from(value.toArray());+ }+ if (typeof value.get === 'function' && typeof value.length === 'number') {+ const out = [];+ for (let i = 0; i < value.length; i++) {+ const valid = typeof value.isValid === 'function' ? value.isValid(i) : true;+ out.push(valid ? jsArrowCell(value.get(i)) : null);+ }+ return out;+ }+ return value;+}++function columnCells(table) {+ const column = table.getChildAt(0);+ return Array.from({length: column.length}, (_, i) => {+ if (typeof column.isValid === 'function' && !column.isValid(i)) {+ return null;+ }+ return jsArrowCell(column.get(i));+ });+}++test('compactArrowTable -> concat does not merge chunks; Vector has no combineChunks', t => {+ const batchA = arrow.tableFromJSON([{lng: -122.4}]);+ const batchB = arrow.tableFromJSON([{lng: -122.5}]);+ const combined = batchA.concat(batchB);+ const column = combined.getChildAt(0);++ t.equal(+ typeof column.combineChunks,+ 'undefined',+ 'apache-arrow JS Vector has concat, not combineChunks'+ );+ t.ok(column.data.length > 1, 'Table.concat keeps one chunk per record batch');+ t.equal(typeof column.concat, 'function', 'Vector.concat exists but does not collapse chunks');++ t.end();+});++test('compactArrowTable -> dictionary, WKB, and nested GeoArrow keep values', t => {+ const geoMeta = metadata => {+ const map = new Map();+ map.set(GEOARROW_METADATA_KEY, metadata);+ return map;+ };++ const dictType = new arrow.Dictionary(new arrow.Utf8(), new arrow.Int8());+ const dictTable = concatColumnTables('city', dictType, ['sf', null], ['nyc']);+ const dictCompacted = compactArrowTable(dictTable, 1);+ t.equal(dictCompacted.batches.length, 1, 'dictionary column should collapse to one batch');+ t.deepEqual(+ columnCells(dictCompacted),+ ['sf', null, 'nyc'],+ 'dictionary values and nulls should survive the Builder rebuild'+ );++ const wkbPoint = (lng, lat) => {+ const bytes = new Uint8Array(21);+ const view = new DataView(bytes.buffer);+ view.setUint8(0, 1);+ view.setUint32(1, 1, true);+ view.setFloat64(5, lng, true);+ view.setFloat64(13, lat, true);+ return bytes;+ };+ const wkbTable = concatColumnTables(+ 'geometry',+ new arrow.Binary(),+ [wkbPoint(-122.4, 37.8), null],+ [wkbPoint(-122.5, 37.9)],+ geoMeta(GEOARROW_EXTENSIONS.WKB)+ );+ wkbTable.schema.metadata.set(+ 'geo',+ JSON.stringify({columns: {geometry: {encoding: 'WKB'}}})+ );+ const wkbCompacted = compactArrowTable(wkbTable, 1);+ t.equal(wkbCompacted.batches.length, 1, 'WKB binary column should collapse to one batch');+ t.deepEqual(+ columnCells(wkbCompacted),+ columnCells(wkbTable),+ 'WKB bytes and nulls should survive the Builder rebuild'+ );+ t.equal(+ wkbCompacted.schema.fields[0].metadata.get(GEOARROW_METADATA_KEY),+ GEOARROW_EXTENSIONS.WKB,+ 'geoarrow.wkb field metadata should be copied'+ );+ t.ok(wkbCompacted.schema.metadata.get('geo'), 'GeoParquet schema metadata should be copied');++ const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false));+ const pointTable = concatColumnTables(+ 'geom',+ pointType,+ [[-122.4, 37.8], null],+ [[-122.5, 37.9]],+ geoMeta(GEOARROW_EXTENSIONS.POINT)+ );+ const pointCompacted = compactArrowTable(pointTable, 1);+ t.equal(pointCompacted.batches.length, 1, 'geoarrow.point should collapse to one batch');+ t.deepEqual(+ columnCells(pointCompacted),+ [[-122.4, 37.8], null, [-122.5, 37.9]],+ 'geoarrow.point coordinates must not become NaN after Builder rebuild'+ );++ const lineType = new arrow.List(new arrow.Field('vertices', pointType, false));+ const lineTable = concatColumnTables(+ 'geom',+ lineType,+ [[[-122.4, 37.8], [-122.5, 37.9]]],+ [[[0, 0], [1, 1]]],+ geoMeta(GEOARROW_EXTENSIONS.LINESTRING)+ );+ const lineCompacted = compactArrowTable(lineTable, 1);+ t.equal(lineCompacted.batches.length, 1, 'geoarrow.linestring should collapse to one batch');+ t.deepEqual(+ columnCells(lineCompacted),+ columnCells(lineTable),+ 'nested geoarrow.linestring coordinates should survive the Builder rebuild'+ );++ const polygonType = new arrow.List(new arrow.Field('rings', lineType, false));+ const ring = [+ [0, 0],+ [1, 0],+ [1, 1],+ [0, 0]+ ];+ const polygonTable = concatColumnTables(+ 'geom',+ polygonType,+ [[ring]],+ [[ring]],+ geoMeta(GEOARROW_EXTENSIONS.POLYGON)+ );+ const polygonCompacted = compactArrowTable(polygonTable, 1);+ t.equal(polygonCompacted.batches.length, 1, 'geoarrow.polygon should collapse to one batch');+ t.deepEqual(+ columnCells(polygonCompacted),+ columnCells(polygonTable),+ 'nested geoarrow.polygon coordinates should survive the Builder rebuild'+ );++ t.end();+});++test('compactArrowTable -> duplicate field names keep both columns', t => {+ const duplicateNameTable = (floatValues, intValues) => {+ const schema = new arrow.Schema([+ new arrow.Field('x', new arrow.Float64()),+ new arrow.Field('x', new arrow.Int32())+ ]);+ const floats = arrow.vectorFromArray(floatValues, new arrow.Float64());+ const ints = arrow.vectorFromArray(intValues, new arrow.Int32());+ const data = arrow.makeData({+ type: new arrow.Struct(schema.fields),+ children: [floats.data[0], ints.data[0]],+ length: floatValues.length+ });+ return new arrow.Table(new arrow.RecordBatch(schema, data));+ };++ const combined = duplicateNameTable([1.5], [1]).concat(duplicateNameTable([2.5], [2]));+ t.equal(combined.numCols, 2, 'fixture should expose both duplicate-named columns');+ t.ok(combined.batches.length > 1, 'fixture should have multiple record batches');++ const compacted = compactArrowTable(combined, 1);+ t.equal(compacted.numCols, 2, 'compaction should not drop a duplicate-named column');+ t.equal(compacted.numRows, 2, 'compaction should keep all rows');+ t.equal(compacted.batches.length, 1, 'duplicate-named columns should still collapse to one batch');+ t.deepEqual(+ [compacted.getChildAt(0).get(0), compacted.getChildAt(0).get(1)],+ [1.5, 2.5],+ 'first duplicate-named column should keep its float values'+ );+ t.deepEqual(+ [compacted.getChildAt(1).get(0), compacted.getChildAt(1).get(1)],+ [1, 2],+ 'second duplicate-named column should keep its int values'+ );++ t.end();+});++test('compactArrowTable -> Builder failure keeps the original table', t => {+ const batchA = arrow.tableFromJSON([{lng: -122.4}]);+ const batchB = arrow.tableFromJSON([{lng: -122.5}]);+ const combined = batchA.concat(batchB);+ const exploding = {+ batches: combined.batches,+ numCols: combined.numCols,+ schema: combined.schema,+ getChildAt() {+ throw new Error('cannot compact');+ }+ };++ const result = compactArrowTable(exploding, 1);+ t.equal(+ result,+ exploding,+ 'compactArrowTable should return the original table when the Builder path throws'+ );++ t.end();+});diff --git a/test/node/utils/filter-utils-test.js b/test/node/utils/filter-utils-test.jsindex 8ef54b5bb..41078920f 100644--- a/test/node/utils/filter-utils-test.js+++ b/test/node/utils/filter-utils-test.js@@ -804,3 +804,117 @@ test('filterUtils -> getPolygonFilterFunctor -> aggregation layers (grid/hexagont.end();});++test('filterUtils -> getPolygonFilterFunctor -> point layer ignores non-finite altitude', t => {+ const squarePolygon = {+ type: 'Feature',+ properties: {},+ geometry: {+ type: 'Polygon',+ coordinates: [+ [+ [-1, -1],+ [1, -1],+ [1, 1],+ [-1, 1],+ [-1, -1]+ ]+ ]+ }+ };++ const filter = {value: squarePolygon};+ const layer = {+ type: 'point',+ config: {columnMode: 'points'},+ getPositionAccessor: () => d => d.position,+ dataToFeature: []+ };++ const fn = getPolygonFilterFunctor(layer, filter, null);++ t.equal(fn({position: [0.5, 0.5, 0]}), true, 'finite altitude should keep a point inside');+ t.equal(+ fn({position: [0.5, 0.5, null]}),+ true,+ 'null altitude should not exclude a valid lng/lat'+ );+ t.equal(+ fn({position: [0.5, 0.5, undefined]}),+ true,+ 'undefined altitude should not exclude a valid lng/lat'+ );+ t.equal(+ fn({position: [10, 10, null]}),+ false,+ 'null altitude should not include a point outside'+ );+ t.equal(+ fn({position: new Float64Array([0.5, 0.5, 0])}),+ true,+ 'typed-array lng/lat should be treated as a valid position'+ );+ t.equal(+ fn({position: [null, 0.5, 0]}),+ false,+ 'null longitude should not coerce to 0 and pass the polygon filter'+ );+ t.equal(+ fn({position: ['', 0.5, 0]}),+ false,+ 'empty-string longitude should not coerce to 0 and pass the polygon filter'+ );++ t.end();+});++test('filterUtils -> getPolygonFilterFunctor -> arc layer validates both endpoints', t => {+ const squarePolygon = {+ type: 'Feature',+ properties: {},+ geometry: {+ type: 'Polygon',+ coordinates: [+ [+ [-1, -1],+ [1, -1],+ [1, 1],+ [-1, 1],+ [-1, -1]+ ]+ ]+ }+ };++ const fn = getPolygonFilterFunctor(+ {+ type: 'arc',+ getPositionAccessor: () => d => d.position+ },+ {value: squarePolygon},+ null+ );++ t.equal(+ fn({position: [0.5, 0.5, 0, 0.2, 0.2, 0]}),+ true,+ 'arc with both endpoints inside should pass'+ );+ t.equal(+ fn({position: [0.5, 0.5, null, 0.2, 0.2, undefined]}),+ true,+ 'null altitude should not exclude a valid arc'+ );+ t.equal(+ fn({position: [0.5, 0.5, 0, null, 0.2, 0]}),+ false,+ 'missing destination longitude should not reach turfPoint'+ );+ t.equal(+ fn({position: [0.5, 0.5, 0, 10, 10, 0]}),+ false,+ 'arc with destination outside should fail'+ );++ t.end();+});