Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix: collapse Arrow record batches above configurable cap (#3663)

Igor Dykhtad810e51cf5cb
diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.ts
index f02db4eb2..4d5c38a0c 100644
--- a/src/duckdb/src/table/duckdb-table.ts
+++ b/src/duckdb/src/table/duckdb-table.ts
@@ -23,6 +23,7 @@ import {
getApplicationConfig,
DatabaseAdapter,
DatabaseConnection,
+ compactArrowTable,
isArrowTable,
isArrowVector
} from '@kepler.gl/utils';
@@ -257,11 +258,13 @@ export class KeplerGlDuckDbTable extends KeplerTable {
restoreGeoarrowMetadata(arrowResult, geoarrowMetadata);
+ const compactedResult = compactArrowTable(arrowResult);
+
fields = useNewFields
- ? arrowSchemaToFields(arrowResult, tableDuckDBTypes)
- : data.fields ?? arrowSchemaToFields(arrowResult, tableDuckDBTypes);
- cols = [...Array(arrowResult.numCols).keys()]
- .map(i => arrowResult.getChildAt(i))
+ ? arrowSchemaToFields(compactedResult, tableDuckDBTypes)
+ : data.fields ?? arrowSchemaToFields(compactedResult, tableDuckDBTypes);
+ cols = [...Array(compactedResult.numCols).keys()]
+ .map(i => compactedResult.getChildAt(i))
.filter(col => col) as arrow.Vector[];
} catch (error) {
console.error('DuckDB table: createTableAndGetArrow', error);
diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts
index eb72f96bf..c5f9c4705 100644
--- a/src/layers/src/geojson-layer/geojson-layer.ts
+++ b/src/layers/src/geojson-layer/geojson-layer.ts
@@ -617,9 +617,33 @@ export default class GeoJsonLayer extends Layer {
const geoField = geoFieldAccessor(this.config.columns)(dataContainer);
// update the latest batch/chunk of geoarrow data when loading data incrementally
- if (geoColumn && geoField && this.dataToFeature.length < dataContainer.numChunks()) {
- // for incrementally loading data, we only load and render the latest batch; otherwise, we will load and render all batches
- const isIncrementalLoad = dataContainer.numChunks() - this.dataToFeature.length === 1;
+ const processedRows = this.centroids.length;
+ const numRows = dataContainer.numRows();
+ const numChunks = dataContainer.numChunks();
+ // WKB (and other whole-column collections) keep dataToFeature.length at 1
+ // even when numChunks() is large. Row count is the progress signal then;
+ // comparing length to numChunks would re-parse WKB on every meta update.
+ const processedWholeColumn =
+ this.dataToFeature.length > 0 && processedRows > 0 && processedRows >= numRows;
+ // Progressive loads keep one dataToFeature entry per chunk, then compact
+ // the finished table to one chunk. Row count already matches, so without
+ // this check we would skip the rebuild and keep excess Deck layers.
+ const compactedAfterProgressiveLoad = this.dataToFeature.length > numChunks;
+ const needsUpdate =
+ this.dataToFeature.length === 0 ||
+ (!processedWholeColumn && this.dataToFeature.length < numChunks) ||
+ (processedRows > 0 && processedRows < numRows) ||
+ compactedAfterProgressiveLoad;
+
+ if (geoColumn && geoField && needsUpdate) {
+ // Incremental only when a new chunk appeared. Compacted tables stay at 1
+ // chunk while row count grows, so those updates reprocess the whole table.
+ const isIncrementalLoad =
+ this.dataToFeature.length > 0 &&
+ !processedWholeColumn &&
+ numChunks - this.dataToFeature.length === 1 &&
+ processedRows > 0 &&
+ processedRows < numRows;
// TODO: add support for COLUMN_MODE_TABLE in getGeojsonLayerMetaFromArrow
const {dataToFeature, bounds, fixedRadius, featureTypes, centroids} =
getGeojsonLayerMetaFromArrow({
@@ -628,9 +652,14 @@ export default class GeoJsonLayer extends Layer {
geoField,
...(isIncrementalLoad ? {chunkIndex: this.dataToFeature.length} : null)
});
- if (centroids) this.centroids = this.centroids.concat(centroids);
+ if (isIncrementalLoad) {
+ if (centroids) this.centroids = this.centroids.concat(centroids);
+ this.dataToFeature = [...this.dataToFeature, ...dataToFeature];
+ } else {
+ this.centroids = centroids || [];
+ this.dataToFeature = dataToFeature;
+ }
this.updateMeta({bounds, fixedRadius, featureTypes});
- this.dataToFeature = [...this.dataToFeature, ...dataToFeature];
}
} else if (this.dataToFeature.length === 0 || this.config.columnMode === COLUMN_MODE_TABLE) {
const getFeature = this.getPositionAccessor(dataContainer);
diff --git a/src/layers/src/heatmap-layer/heatmap-layer.ts b/src/layers/src/heatmap-layer/heatmap-layer.ts
index eabf1289c..f5685ae74 100644
--- a/src/layers/src/heatmap-layer/heatmap-layer.ts
+++ b/src/layers/src/heatmap-layer/heatmap-layer.ts
@@ -30,7 +30,7 @@ import {booleanWithin} from '@turf/boolean-within';
import {point as turfPoint} from '@turf/helpers';
import {Feature, Polygon} from 'geojson';
-import {getGeoArrowPointLayerProps, FindDefaultLayerPropsReturnValue} from '../layer-utils';
+import {getGeoArrowPointLayerProps, FindDefaultLayerPropsReturnValue, getGeoArrowPointCoords} from '../layer-utils';
import {getFilterDataFunc} from '../aggregation-layer';
import {
parseGeoJsonRawFeature,
@@ -88,7 +88,8 @@ export const geoarrowPosAccessor =
(dc: DataContainerInterface) =>
(d: {index: number}): number[] => {
const row = dc.valueAt(d.index, geoarrow.fieldIdx);
- return [row.get(0), row.get(1)];
+ const coords = getGeoArrowPointCoords(row);
+ return [coords[0], coords[1]];
};
export const geojsonAccessor =
diff --git a/src/layers/src/layer-utils.ts b/src/layers/src/layer-utils.ts
index 72ab21bca..ee074093e 100644
--- a/src/layers/src/layer-utils.ts
+++ b/src/layers/src/layer-utils.ts
@@ -24,14 +24,17 @@ import {convertGeoArrowGeometryToGeoJSON} from '@loaders.gl/gis';
import {WKBLoader} from '@loaders.gl/wkt';
import {geojsonToBinary} from '@loaders.gl/gis';
import {
- BinaryFeatureCollection,
Geometry,
BinaryPointFeature,
BinaryLineFeature,
BinaryPolygonFeature
} from '@loaders.gl/schema';
-import {DeckGlGeoTypes, GeojsonDataMaps} from './geojson-layer/geojson-utils';
+import {
+ DeckGlGeoTypes,
+ GeojsonDataMaps,
+ getCentroidFromGeometry
+} from './geojson-layer/geojson-utils';
export type FindDefaultLayerProps = {
label: string;
@@ -87,7 +90,7 @@ function getBinaryGeometriesFromWKBArrow(
geoColumn: arrow.Vector,
options: {chunkIndex?: number; chunkOffset?: number}
): GeojsonLayerMetaProps {
- const dataToFeature: BinaryFeatureCollection[] = [];
+ const centroids: Array<number[] | null> = [];
const featureTypes: GeojsonLayerMetaProps['featureTypes'] = {
point: false,
line: false,
@@ -102,8 +105,8 @@ function getBinaryGeometriesFromWKBArrow(
let featureIndex = globalFeatureIdOffset;
let bounds: [number, number, number, number] = [Infinity, Infinity, -Infinity, -Infinity];
+ const geojsonFeatures: Feature[] = [];
chunks.forEach(chunk => {
- const geojsonFeatures: Feature[] = [];
for (let i = 0; i < chunk.length; ++i) {
// ignore features without any geometry
if (chunk.valueOffsets[i + 1] - chunk.valueOffsets[i] > 0) {
@@ -118,54 +121,57 @@ function getBinaryGeometriesFromWKBArrow(
properties: {index: featureIndex}
};
geojsonFeatures.push(feature);
+ centroids.push(getCentroidFromGeometry(geometry));
const {type} = geometry;
featureTypes.polygon = type === 'Polygon' || type === 'MultiPolygon';
featureTypes.point = type === 'Point' || type === 'MultiPoint';
featureTypes.line = type === 'LineString' || type === 'MultiLineString';
+ } else {
+ centroids.push(null);
}
featureIndex++;
}
+ });
- const geojsonToBinaryOptions = {
- triangulate: true,
- fixRingWinding: true
- };
- const binaryFeatures = geojsonToBinary(geojsonFeatures, geojsonToBinaryOptions);
-
- // Need to update globalFeatureIds, to take into account previous batches,
- // as geojsonToBinary doesn't have such option.
- const featureTypesArr = ['points', 'lines', 'polygons'];
- featureTypesArr.forEach(prop => {
- const features = binaryFeatures[prop] as
- | BinaryPointFeature
- | BinaryLineFeature
- | BinaryPolygonFeature;
- if (features) {
- bounds = updateBoundsFromGeoArrowSamples(
- features.positions.value as Float64Array,
- features.positions.size,
- bounds
- );
-
- const {globalFeatureIds, numericProps} = features;
- const {index} = numericProps;
- const len = globalFeatureIds.value.length;
- for (let i = 0; i < len; ++i) {
- globalFeatureIds.value[i] = index.value[i];
- }
+ // One BinaryFeatureCollection for the whole column. A collection per Arrow
+ // chunk becomes one GeoJsonLayer (with fill/stroke/point sublayers) each, and
+ // parquet/DuckDB files easily exceed deck.gl's 255 pickable-layer cap.
+ const geojsonToBinaryOptions = {
+ triangulate: true,
+ fixRingWinding: true
+ };
+ const binaryFeatures = geojsonToBinary(geojsonFeatures, geojsonToBinaryOptions);
+
+ const featureTypesArr = ['points', 'lines', 'polygons'];
+ featureTypesArr.forEach(prop => {
+ const features = binaryFeatures[prop] as
+ | BinaryPointFeature
+ | BinaryLineFeature
+ | BinaryPolygonFeature;
+ if (features) {
+ bounds = updateBoundsFromGeoArrowSamples(
+ features.positions.value as Float64Array,
+ features.positions.size,
+ bounds
+ );
+
+ const {globalFeatureIds, numericProps} = features;
+ const {index} = numericProps;
+ const len = globalFeatureIds.value.length;
+ for (let i = 0; i < len; ++i) {
+ globalFeatureIds.value[i] = index.value[i];
}
- });
-
- dataToFeature.push(binaryFeatures);
+ }
});
return {
- dataToFeature: dataToFeature,
+ dataToFeature: [binaryFeatures],
featureTypes: featureTypes,
bounds,
- fixedRadius: false
+ fixedRadius: false,
+ centroids
};
}
@@ -290,6 +296,38 @@ export function getGeoArrowPointFields(fields: Field[]): Field[] {
});
}
+/**
+ * Reads lng/lat from a GeoArrow point cell. Arrow may return a FixedSizeList
+ * (with .get), a typed array, a plain [lng, lat] array, or a {x,y} struct.
+ */
+export function getGeoArrowPointCoords(row: any): [number, number, number] {
+ if (row == null) {
+ return [NaN, NaN, 0];
+ }
+ if (typeof row.get === 'function') {
+ const x = row.get(0) ?? row.get('x') ?? row.get('lng') ?? row.get('lon');
+ const y = row.get(1) ?? row.get('y') ?? row.get('lat');
+ if (Number.isFinite(x) && Number.isFinite(y)) {
+ return [Number(x), Number(y), 0];
+ }
+ }
+ if (typeof row.toArray === 'function') {
+ const arr = row.toArray();
+ return [Number(arr[0]), Number(arr[1]), 0];
+ }
+ if (Array.isArray(row) || ArrayBuffer.isView(row)) {
+ return [Number(row[0]), Number(row[1]), 0];
+ }
+ if (typeof row === 'object') {
+ const x = row.x ?? row.lng ?? row.lon;
+ const y = row.y ?? row.lat;
+ if (x !== undefined && y !== undefined) {
+ return [Number(x), Number(y), 0];
+ }
+ }
+ return [NaN, NaN, 0];
+}
+
/**
* Builds an arrow vector compatible with ARROW:extension:name geoarrow.point.
* @param getPosition Position accessor.
@@ -305,7 +343,6 @@ export function createGeoArrowPointVector(
// in a correct arrow format, as this approach seems too excessive for just a simple interleaved buffer.
const numElements = dataContainer.numRows();
- const table = dataContainer.getTable();
const numCoords = numElements > 0 ? getPosition({index: 0}).length : 2;
const precision = 2;
@@ -319,21 +356,16 @@ export function createGeoArrowPointVector(
const fixedSizeListBuilder = new arrow.FixedSizeListBuilder({type: fixedSizeList});
fixedSizeListBuilder.addChild(floatBuilder);
- const assembledBatches: arrow.Data[] = [];
+ // One record batch, not one per source batch: a batch per parquet/DuckDB
+ // chunk would create hundreds of pickable ScatterplotLayers and hit deck.gl's
+ // 255-layer picking cap.
const indexData = {index: 0};
- for (let batchIndex = 0; batchIndex < table.batches.length; ++batchIndex) {
- const numRowsInBatch = table.batches[batchIndex].numRows;
-
- for (let i = 0; i < numRowsInBatch; ++i) {
- const pos = getPosition(indexData);
- fixedSizeListBuilder.append(pos);
-
- ++indexData.index;
- }
- assembledBatches.push(fixedSizeListBuilder.flush());
+ for (let i = 0; i < numElements; ++i) {
+ indexData.index = i;
+ fixedSizeListBuilder.append(getPosition(indexData));
}
- return arrow.makeVector(assembledBatches);
+ return arrow.makeVector(numElements > 0 ? [fixedSizeListBuilder.flush()] : []);
}
/**
diff --git a/src/layers/src/point-layer/point-layer.ts b/src/layers/src/point-layer/point-layer.ts
index 5ab058bf9..5045cca98 100644
--- a/src/layers/src/point-layer/point-layer.ts
+++ b/src/layers/src/point-layer/point-layer.ts
@@ -41,7 +41,8 @@ import {
createGeoArrowPointVector,
getFilteredIndex,
getNeighbors,
- FindDefaultLayerProps
+ FindDefaultLayerProps,
+ getGeoArrowPointCoords
} from '../layer-utils';
import {getGeojsonPointDataMaps, GeojsonPointDataMaps} from '../geojson-layer/geojson-utils';
import {
@@ -131,7 +132,7 @@ export const geoarrowPosAccessor =
(dataContainer: DataContainerInterface) =>
(d: {index: number}) => {
const row = dataContainer.valueAt(d.index, geoarrow.fieldIdx);
- return [row.get(0), row.get(1), 0];
+ return getGeoArrowPointCoords(row);
};
export const COLUMN_MODE_POINTS = 'points';
diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.ts
index 2759426b4..17b9c011f 100644
--- a/src/processors/src/data-processor.ts
+++ b/src/processors/src/data-processor.ts
@@ -20,6 +20,7 @@ import {ProcessorResult, Field} from '@kepler.gl/types';
import {
arrowDataTypeToAnalyzerDataType,
arrowDataTypeToFieldType,
+ compactArrowTable,
hasOwnProperty,
isPlainObject
} from '@kepler.gl/utils';
@@ -616,17 +617,34 @@ function castBigIntColumnsToFloat64(arrowTable: arrow.Table): arrow.Table {
return new arrow.Table(newColumns);
}
+export type ProcessArrowBatchesOptions = {
+ /**
+ * When false, skip compactArrowTable. Progressive Arrow loading uses this so
+ * each incoming batch does not copy the accumulated table. Defaults to true.
+ * The final processFileContent call leaves this unset so the completed table
+ * is compacted once.
+ */
+ compact?: boolean;
+};
+
/**
* Parse arrow batches returned from parseInBatches()
*
- * @param arrowTable the arrow table to parse
+ * @param arrowBatches the arrow record batches to parse
+ * @param options optional processing flags
* @returns dataset containing `fields` and `rows` or null
*/
-export function processArrowBatches(arrowBatches: arrow.RecordBatch[]): ProcessorResult | null {
+export function processArrowBatches(
+ arrowBatches: arrow.RecordBatch[],
+ options?: ProcessArrowBatchesOptions
+): ProcessorResult | null {
if (arrowBatches.length === 0) {
return null;
}
- const arrowTable = castBigIntColumnsToFloat64(new arrow.Table(arrowBatches));
+ const arrowTable =
+ options?.compact === false
+ ? castBigIntColumnsToFloat64(new arrow.Table(arrowBatches))
+ : compactArrowTable(castBigIntColumnsToFloat64(new arrow.Table(arrowBatches)));
const fields = arrowSchemaToFields(arrowTable);
const cols = [...Array(arrowTable.numCols).keys()].map(i => arrowTable.getChildAt(i));
diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.ts
index 2e229c93b..8d090cb26 100644
--- a/src/processors/src/file-handler.ts
+++ b/src/processors/src/file-handler.ts
@@ -70,6 +70,11 @@ export type ProcessFileDataContent = {
sourceUrl?: string;
/** User-selected or inferred remote file format (csv, geojson, parquet, …). */
keplerFormat?: string;
+ /**
+ * When true, skip Arrow record-batch compaction. Set on intermediate
+ * progressive Arrow loads so each batch does not copy the growing table.
+ */
+ skipArrowCompact?: boolean;
};
export {isArrowTable};
@@ -279,7 +284,10 @@ export async function processFileData({
// eslint-disable-next-line no-useless-catch
let result;
try {
- result = await processor(data);
+ result =
+ format === DATASET_FORMATS.arrow && content.skipArrowCompact
+ ? await processArrowBatches(data as any, {compact: false})
+ : await processor(data);
} catch (error) {
throw new Error(`Can not process uploaded file, ${getError(error as Error)}`);
}
diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts
index d037d9186..cfa4e5ca6 100644
--- a/src/reducers/src/vis-state-updaters.ts
+++ b/src/reducers/src/vis-state-updaters.ts
@@ -3517,7 +3517,12 @@ export const nextFileBatchUpdater = (
fileName.endsWith('arrow') &&
accumulated?.data?.length > 0
? [
- PROCESS_FILE_DATA({content: accumulated, fileCache: []}).bimap(
+ // Skip compaction while batches are still arriving. The completed
+ // file is compacted once in processFileContent.
+ PROCESS_FILE_DATA({
+ content: {...accumulated, skipArrowCompact: true},
+ fileCache: []
+ }).bimap(
result => loadFilesSuccess(result),
err => loadFilesErr(fileName, err)
)
diff --git a/src/utils/src/application-config.ts b/src/utils/src/application-config.ts
index 0245f4057..d5cd5b05e 100644
--- a/src/utils/src/application-config.ts
+++ b/src/utils/src/application-config.ts
@@ -224,6 +224,22 @@ export type KeplerApplicationConfig = {
/** Controls Redux action logging verbosity in dev mode. See {@link ReduxLogLevel}. */
reduxLogLevel?: ReduxLogLevel;
+
+ /**
+ * If an Arrow/Parquet table has more record batches than this, they are
+ * compacted into a single batch. The default is `1`, so any table with more
+ * than one record batch is compacted. Deck.gl picking supports at most 255
+ * pickable leaf layers, and GeoJSON adds fill/stroke/point sublayers per
+ * batch, so a higher cap still overruns picking. Compacting copies the table.
+ * Progressive Arrow loading skips compaction on intermediate batches and
+ * compacts the completed file once. Set a very large number to never compact.
+ *
+ * @example
+ * ```
+ * initApplicationConfig({maxArrowBatches: 64});
+ * ```
+ */
+ maxArrowBatches?: number;
};
const DEFAULT_APPLICATION_CONFIG: Required<KeplerApplicationConfig> = {
@@ -319,7 +335,9 @@ const DEFAULT_APPLICATION_CONFIG: Required<KeplerApplicationConfig> = {
customIconUrl: '',
- reduxLogLevel: 1
+ reduxLogLevel: 1,
+
+ maxArrowBatches: 1
};
const applicationConfig: Required<KeplerApplicationConfig> = DEFAULT_APPLICATION_CONFIG;
diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.ts
index f462b11db..698bf0e18 100644
--- a/src/utils/src/arrow-data-container.ts
+++ b/src/utils/src/arrow-data-container.ts
@@ -7,6 +7,7 @@ import * as arrow from 'apache-arrow';
import {console as globalConsole} from 'global/window';
import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer';
+import {getApplicationConfig} from './application-config';
import {DataContainerInterface, RangeOptions} from './data-container-interface';
import {DataRow, SharedRowOptions} from './data-row';
@@ -120,6 +121,185 @@ function* columnIterator(dataContainer: DataContainerInterface, columnIndex: num
}
}
+const DEFAULT_MAX_ARROW_BATCHES = 1;
+
+/**
+ * Collapse Arrow record batches into as few chunks as possible.
+ *
+ * Deck.gl picking encodes at most 255 pickable leaf layers. Kepler creates one
+ * Deck layer per Arrow record batch (and GeoJsonLayer adds fill/stroke/point
+ * sublayers), so parquet/DuckDB tables with hundreds of batches make hover and
+ * click miss most of the data. Combining chunks keeps picking under that cap
+ * without changing row values.
+ *
+ * apache-arrow JS does not expose Vector.combineChunks in the version Kepler
+ * uses, so we rebuild each column through a Builder (one chunk per column).
+ *
+ * Compaction only runs when `table.batches.length` is greater than
+ * `maxArrowBatches` from {@link getApplicationConfig} (default `1`) or the
+ * optional override. Progressive Arrow loading does not compact on each
+ * incoming batch; {@link ArrowDataContainer.update} assigns new chunks as-is
+ * and the completed file is compacted once.
+ */
+export function compactArrowTable(table: arrow.Table, maxArrowBatches?: number): arrow.Table {
+ const batchLimit =
+ maxArrowBatches ?? getApplicationConfig().maxArrowBatches ?? DEFAULT_MAX_ARROW_BATCHES;
+
+ if (!table || !Array.isArray(table.batches) || table.batches.length <= 1) {
+ return table;
+ }
+
+ if (table.batches.length <= batchLimit) {
+ return table;
+ }
+
+ try {
+ const columns: Record<string, arrow.Vector> = {};
+ const seenNames = new Map<string, number>();
+ for (let i = 0; i < table.numCols; i++) {
+ const field = table.schema.fields[i];
+ const column = table.getChildAt(i);
+ if (!column) {
+ continue;
+ }
+ // Arrow schemas may repeat field names; object keys would drop earlier columns.
+ const seen = seenNames.get(field.name) ?? 0;
+ seenNames.set(field.name, seen + 1);
+ const key = seen === 0 ? field.name : `${field.name}_${seen}`;
+ columns[key] = compactArrowVector(column, column.type || field.type);
+ }
+
+ // Always build a kepler apache-arrow Table. Using `table.constructor` can
+ // mix DuckDB's bundled Arrow Table with rebuilt vectors and drop schema
+ // metadata (`geo`, ARROW:extension:name) that Kepler needs to create layers.
+ const compacted = tableFromCompactedColumns(table, columns);
+
+ if (compacted?.batches?.length && compacted.batches.length < table.batches.length) {
+ return compacted;
+ }
+ } catch {
+ // Keep the original table if this Arrow build cannot combine chunks.
+ }
+
+ return table;
+}
+
+function copyArrowSchemaMetadata(from?: arrow.Schema | null, to?: arrow.Schema | null): void {
+ if (!from || !to) {
+ return;
+ }
+
+ const fromMeta = from.metadata as Map<string, string> | undefined;
+ const toMeta = to.metadata as Map<string, string> | undefined;
+ if (fromMeta && toMeta && typeof fromMeta.forEach === 'function' && typeof toMeta.set === 'function') {
+ fromMeta.forEach((value, key) => {
+ toMeta.set(key, value);
+ });
+ }
+
+ const fromByName = new Map((from.fields || []).map(field => [field.name, field]));
+ (to.fields || []).forEach((toField, index) => {
+ const fromField = from.fields?.[index] || fromByName.get(toField.name);
+ const src = fromField?.metadata as Map<string, string> | undefined;
+ const dst = toField?.metadata as Map<string, string> | undefined;
+ if (!src || !dst || typeof src.forEach !== 'function' || typeof dst.set !== 'function') {
+ return;
+ }
+ src.forEach((value, key) => {
+ dst.set(key, value);
+ });
+ });
+}
+
+function tableFromCompactedColumns(
+ source: arrow.Table,
+ columns: Record<string, arrow.Vector>
+): arrow.Table {
+ let compacted: arrow.Table;
+ try {
+ compacted = new arrow.Table(source.schema, columns);
+ } catch {
+ compacted = new arrow.Table(columns);
+ }
+ copyArrowSchemaMetadata(source.schema, compacted.schema);
+ return compacted;
+}
+
+/**
+ * Convert an Arrow cell from Vector.get() into a value makeBuilder.append accepts.
+ * Nested GeoArrow (FixedSizeList / List) cells are Vectors, not arrays.
+ */
+function arrowCellToBuilderValue(value: unknown): unknown {
+ if (value == null) {
+ return null;
+ }
+ if (ArrayBuffer.isView(value)) {
+ return Array.from(value as any);
+ }
+
+ const nested = value as {
+ toArray?: () => ArrayLike<unknown>;
+ numChildren?: number;
+ get?: (index: number) => unknown;
+ length?: number;
+ isValid?: (index: number) => boolean;
+ };
+
+ // Primitive numeric child of a FixedSizeList (lng/lat).
+ if (typeof nested.toArray === 'function' && nested.numChildren === 0) {
+ return Array.from(nested.toArray());
+ }
+
+ if (typeof nested.get === 'function' && typeof nested.length === 'number') {
+ const out: unknown[] = [];
+ for (let i = 0; i < nested.length; i++) {
+ const valid = typeof nested.isValid === 'function' ? nested.isValid(i) : true;
+ out.push(valid ? arrowCellToBuilderValue(nested.get(i)) : null);
+ }
+ return out;
+ }
+
+ return value;
+}
+
+function compactArrowVector(column: arrow.Vector, type: arrow.DataType = column.type): arrow.Vector {
+ if (!column || !column.data || column.data.length <= 1) {
+ return column;
+ }
+
+ const maybeCombine = (column as {combineChunks?: () => arrow.Vector}).combineChunks;
+ if (typeof maybeCombine === 'function') {
+ try {
+ const combined = maybeCombine.call(column);
+ if (combined?.data?.length <= 1) {
+ return combined;
+ }
+ } catch {
+ // fall through to Builder
+ }
+ }
+
+ const builder = arrow.makeBuilder({type, nullValues: [null]});
+ const length = column.length;
+ for (let i = 0; i < length; i++) {
+ const isValid = typeof (column as {isValid?: (index: number) => boolean}).isValid === 'function'
+ ? (column as {isValid: (index: number) => boolean}).isValid(i)
+ : true;
+ // FixedSizeList/List .get() returns nested Vectors. Appending those
+ // silently writes NaN (points) or throws (lines/polygons). Plain JS
+ // arrays are what makeBuilder expects.
+ builder.append(isValid ? arrowCellToBuilderValue(column.get(i)) : null);
+ }
+ const finished = builder.finish() as arrow.Vector | {toVector: () => arrow.Vector};
+ if (finished instanceof arrow.Vector) {
+ return finished;
+ }
+ if (typeof (finished as {toVector?: () => arrow.Vector}).toVector === 'function') {
+ return (finished as {toVector: () => arrow.Vector}).toVector();
+ }
+ return column;
+}
+
/**
* A data container where all data is stored in raw Arrow table
*/
@@ -144,14 +324,16 @@ export class ArrowDataContainer implements DataContainerInterface {
throw Error("ArrowDataContainer: columns object isn't an array");
}
- this._cols = data.cols;
- this._numColumns = data.cols.length;
- this._numRows = data.cols[0].length;
this._fields = data.fields || [];
- this._numChunks = data.cols[0].data.length;
- // this._colData = data.cols.map(c => c.toArray());
-
- this._arrowTable = data.arrowTable || this._createTable();
+ this._cols = data.cols;
+ this._numColumns = this._cols.length;
+ this._numRows = this._cols[0]?.length ?? 0;
+ this._numChunks = this._cols[0]?.data?.length ?? 0;
+ const table = compactArrowTable(data.arrowTable || this._createTable());
+ this._arrowTable = table;
+ if (table.numCols > 0) {
+ this._assignTable(table);
+ }
}
/**
@@ -170,20 +352,36 @@ export class ArrowDataContainer implements DataContainerInterface {
return this._arrowTable;
}
+ private _assignTable(table: arrow.Table) {
+ this._arrowTable = table;
+ this._cols = Array.from(
+ {length: table.numCols},
+ (_, i) => table.getChildAt(i) as arrow.Vector
+ ).filter(col => col);
+ this._numColumns = this._cols.length;
+ this._numRows = this._cols[0]?.length ?? 0;
+ this._numChunks = this._cols[0]?.data?.length ?? 0;
+ }
+
update(updateData: arrow.Vector<any>[] | arrow.Table) {
+ // Incremental Arrow loads append batches here. Compacting on every update
+ // would copy the growing table (Builder over every cell). Skip collapse;
+ // processArrowBatches compact once when the file is fully loaded.
const isArrow = isArrowTable(updateData);
if (isArrow) {
- this._cols = Array.from(
- {length: updateData.numCols},
- (_, i) => updateData.getChildAt(i) as arrow.Vector
- ).filter(col => col);
+ this._assignTable(updateData);
} else {
this._cols = updateData;
+ const table = this._createTable();
+ if (table.numCols > 0) {
+ this._assignTable(table);
+ } else {
+ this._numColumns = this._cols.length;
+ this._numRows = this._cols[0]?.length ?? 0;
+ this._numChunks = this._cols[0]?.data?.length ?? 0;
+ this._arrowTable = table;
+ }
}
- this._numColumns = this._cols?.length ?? 0;
- this._numRows = this._cols?.[0]?.length ?? 0;
- this._numChunks = this._cols?.[0]?.data?.length ?? 0;
- this._arrowTable = isArrow ? updateData : this._createTable();
// cache column data to make valueAt() faster
// this._colData = this._cols.map(c => c.toArray());
diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.ts
index 954031a1e..a3ab46ede 100644
--- a/src/utils/src/filter-utils.ts
+++ b/src/utils/src/filter-utils.ts
@@ -376,6 +376,21 @@ export function getFilterProps(
}
}
+function hasFiniteLngLat(pos: unknown): pos is number[] {
+ if (pos == null || !(Array.isArray(pos) || ArrayBuffer.isView(pos))) {
+ return false;
+ }
+ const values = pos as ArrayLike<unknown>;
+ // Do not coerce with Number(): Number(null/''/false) is 0 and would pass.
+ return (
+ values.length >= 2 &&
+ typeof values[0] === 'number' &&
+ Number.isFinite(values[0]) &&
+ typeof values[1] === 'number' &&
+ Number.isFinite(values[1])
+ );
+}
+
export const getPolygonFilterFunctor = (layer, filter, dataContainer) => {
const getPosition = layer.getPositionAccessor(dataContainer);
@@ -388,22 +403,15 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => {
if (!coordinates) return false;
if (Array.isArray(coordinates[0])) {
return (coordinates as number[][]).some(
- coord =>
- coord.length >= 2 &&
- coord.every(Number.isFinite) &&
- isInPolygon(coord, filter.value)
+ coord => hasFiniteLngLat(coord) && isInPolygon(coord, filter.value)
);
}
- return (
- coordinates.length >= 2 &&
- coordinates.every(Number.isFinite) &&
- isInPolygon(coordinates, filter.value)
- );
+ return hasFiniteLngLat(coordinates) && isInPolygon(coordinates, filter.value);
};
}
return data => {
const pos = getPosition(data);
- return pos.every(Number.isFinite) && isInPolygon(pos, filter.value);
+ return hasFiniteLngLat(pos) && isInPolygon(pos, filter.value);
};
case LAYER_TYPES.grid:
case LAYER_TYPES.hexagon:
@@ -418,14 +426,15 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => {
}
return data => {
const pos = getPosition(data);
- return pos.every(Number.isFinite) && isInPolygon(pos, filter.value);
+ return hasFiniteLngLat(pos) && isInPolygon(pos, filter.value);
};
case LAYER_TYPES.arc:
case LAYER_TYPES.line:
return data => {
const pos = getPosition(data);
return (
- pos.every(Number.isFinite) &&
+ hasFiniteLngLat(pos) &&
+ hasFiniteLngLat([pos[3], pos[4]]) &&
[
[pos[0], pos[1]],
[pos[3], pos[4]]
@@ -446,7 +455,7 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => {
return false;
}
const pos = getCentroid({id});
- return pos.every(Number.isFinite) && isInPolygon(pos, filter.value);
+ return hasFiniteLngLat(pos) && isInPolygon(pos, filter.value);
};
case LAYER_TYPES.geojson:
return data => {
@@ -461,7 +470,7 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => {
}
return data => {
const pos = getPosition(data);
- return pos.every(Number.isFinite) && isInPolygon(pos, filter.value);
+ return hasFiniteLngLat(pos) && isInPolygon(pos, filter.value);
};
default:
return () => true;
@@ -830,7 +839,9 @@ export function isInRange(val: any, domain: number[]): boolean {
* @return {boolean}
*/
export function isInPolygon(point: number[], polygon: any): boolean {
- return booleanWithin(turfPoint(point), polygon);
+ // turfPoint requires a plain array; Arrow accessors can return typed arrays
+ const lngLat = Array.isArray(point) ? point : [Number(point[0]), Number(point[1])];
+ return booleanWithin(turfPoint(lngLat), polygon);
}
export function getTimeWidgetTitleFormatter(domain: [number, number]): string | null {
if (!isValidTimeDomain(domain)) {
diff --git a/src/utils/src/index.ts b/src/utils/src/index.ts
index b38a85654..f730968ee 100644
--- a/src/utils/src/index.ts
+++ b/src/utils/src/index.ts
@@ -144,6 +144,7 @@ export * from './map-utils';
export {
ArrowDataContainer,
+ compactArrowTable,
arrowDataTypeToAnalyzerDataType,
arrowDataTypeToFieldType,
isArrowTable,
diff --git a/test/node/processors/file-handler-test.js b/test/node/processors/file-handler-test.js
index fdb6cd782..b190b92fb 100644
--- a/test/node/processors/file-handler-test.js
+++ b/test/node/processors/file-handler-test.js
@@ -2,7 +2,8 @@
// Copyright contributors to the kepler.gl project
import test from 'tape';
-import {isKeplerGlMap, makeProgressIterator, filesToDataPayload, processFileData} from '@kepler.gl/processors';
+import {isKeplerGlMap, makeProgressIterator, filesToDataPayload, processFileData, processArrowBatches} from '@kepler.gl/processors';
+import * as arrow from 'apache-arrow';
import {parsedFields, parsedRows} from 'test/fixtures/row-object';
import {
savedStateV1InteractionCoordinate as keplerglMap,
@@ -277,3 +278,59 @@ test('#file-handler -> processFileData remote ids do not collide on filename', a
t.end();
});
+
+test('#file-handler -> processArrowBatches skip compact for incremental loads', t => {
+ const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]);
+ const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]);
+ const combined = batchA.concat(batchB);
+
+ t.ok(combined.batches.length > 1, 'fixture should have multiple record batches');
+
+ const skipped = processArrowBatches(combined.batches, {compact: false});
+ t.equal(
+ skipped.cols[0].data.length,
+ combined.batches.length,
+ 'compact:false should leave record batches unchanged'
+ );
+
+ const compacted = processArrowBatches(combined.batches);
+ t.equal(compacted.cols[0].data.length, 1, 'default processArrowBatches should compact');
+ t.equal(compacted.cols[0].length, 2, 'compacted table should keep all rows');
+
+ t.end();
+});
+
+test('#file-handler -> processFileData skipArrowCompact', async t => {
+ const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]);
+ const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]);
+ const combined = batchA.concat(batchB);
+
+ const incremental = await processFileData({
+ content: {
+ fileName: 'points.arrow',
+ data: combined.batches,
+ skipArrowCompact: true
+ },
+ fileCache: []
+ });
+ t.equal(
+ incremental[0].data.cols[0].data.length,
+ combined.batches.length,
+ 'progressive processFileData should not compact Arrow batches'
+ );
+
+ const finished = await processFileData({
+ content: {
+ fileName: 'points.arrow',
+ data: combined.batches
+ },
+ fileCache: []
+ });
+ t.equal(
+ finished[0].data.cols[0].data.length,
+ 1,
+ 'final processFileData should compact Arrow batches'
+ );
+
+ t.end();
+});
diff --git a/test/node/utils/data-container-test.js b/test/node/utils/data-container-test.js
index 200be5c68..7095ef48e 100644
--- a/test/node/utils/data-container-test.js
+++ b/test/node/utils/data-container-test.js
@@ -3,7 +3,9 @@
import test from 'tape';
-import {createDataContainer, createIndexedDataContainer} from '@kepler.gl/utils';
+import {createDataContainer, createIndexedDataContainer, compactArrowTable, ArrowDataContainer} from '@kepler.gl/utils';
+import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants';
+import * as arrow from 'apache-arrow';
const data = [
[10, 20], // 0
@@ -120,3 +122,321 @@ test('IndexedDataContainer', t => {
t.end();
});
+
+test('ArrowDataContainer -> compactArrowTable collapses record batches over the limit', t => {
+ const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]);
+ const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]);
+ const combined = batchA.concat(batchB);
+
+ t.ok(combined.batches.length > 1, 'fixture should have multiple record batches');
+
+ const belowLimit = compactArrowTable(combined, 255);
+ t.equal(
+ belowLimit.batches.length,
+ combined.batches.length,
+ 'compactArrowTable should leave tables at or under maxArrowBatches unchanged'
+ );
+
+ combined.schema.metadata.set(
+ 'geo',
+ JSON.stringify({columns: {lng: {encoding: 'WKB'}}})
+ );
+ combined.schema.fields[0].metadata.set('ARROW:extension:name', 'geoarrow.wkb');
+
+ const compacted = compactArrowTable(combined, 1);
+ t.equal(compacted.numRows, 2, 'compactArrowTable should keep all rows');
+ t.equal(compacted.batches.length, 1, 'compactArrowTable should collapse into one batch');
+ t.ok(
+ compacted.schema.metadata.get('geo'),
+ 'compactArrowTable should keep GeoParquet schema metadata used to create layers'
+ );
+ t.equal(
+ compacted.schema.fields[0].metadata.get('ARROW:extension:name'),
+ 'geoarrow.wkb',
+ 'compactArrowTable should keep geoarrow field metadata'
+ );
+
+ const cols = Array.from({length: combined.numCols}, (_, i) => combined.getChildAt(i)).filter(
+ Boolean
+ );
+ const arrowDc = new ArrowDataContainer({
+ cols,
+ fields: combined.schema.fields.map((field, fieldIdx) => ({
+ name: field.name,
+ fieldIdx
+ })),
+ arrowTable: compactArrowTable(combined, 1)
+ });
+
+ t.equal(arrowDc.numRows(), 2, 'ArrowDataContainer should keep all rows');
+ t.equal(arrowDc.numChunks(), 1, 'ArrowDataContainer should store a compacted table');
+ t.deepEqual(arrowDc.valueAt(0, 0), -122.4, 'compacted container should preserve values');
+
+ t.end();
+});
+
+test('ArrowDataContainer -> update does not collapse record batches', t => {
+ const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]);
+ const colsA = Array.from({length: batchA.numCols}, (_, i) => batchA.getChildAt(i)).filter(Boolean);
+ const dc = new ArrowDataContainer({
+ cols: colsA,
+ fields: batchA.schema.fields.map((field, fieldIdx) => ({
+ name: field.name,
+ fieldIdx
+ })),
+ arrowTable: batchA
+ });
+
+ const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]);
+ const combined = batchA.concat(batchB);
+ dc.update(combined);
+
+ t.equal(dc.numRows(), 2, 'update should keep all rows');
+ t.equal(
+ dc.numChunks(),
+ combined.batches.length,
+ 'update should not compact batches during incremental loading'
+ );
+ t.deepEqual(dc.valueAt(1, 0), -122.5, 'update should preserve newly appended values');
+
+ t.end();
+});
+
+function finishArrowVector(builder) {
+ const finished = builder.finish();
+ if (finished instanceof arrow.Vector) {
+ return finished;
+ }
+ return finished.toVector();
+}
+
+function tableFromColumnValues(name, type, values, metadata) {
+ const builder = arrow.makeBuilder({type, nullValues: [null]});
+ values.forEach(value => builder.append(value));
+ const vector = finishArrowVector(builder);
+ const field = new arrow.Field(name, type, true, metadata);
+ return new arrow.Table(new arrow.Schema([field]), {[name]: vector});
+}
+
+function concatColumnTables(name, type, leftValues, rightValues, metadata) {
+ return tableFromColumnValues(name, type, leftValues, metadata).concat(
+ tableFromColumnValues(name, type, rightValues, metadata)
+ );
+}
+
+function jsArrowCell(value) {
+ if (value == null) {
+ return null;
+ }
+ if (ArrayBuffer.isView(value)) {
+ return Array.from(value);
+ }
+ if (typeof value.toArray === 'function' && value.numChildren === 0) {
+ return Array.from(value.toArray());
+ }
+ if (typeof value.get === 'function' && typeof value.length === 'number') {
+ const out = [];
+ for (let i = 0; i < value.length; i++) {
+ const valid = typeof value.isValid === 'function' ? value.isValid(i) : true;
+ out.push(valid ? jsArrowCell(value.get(i)) : null);
+ }
+ return out;
+ }
+ return value;
+}
+
+function columnCells(table) {
+ const column = table.getChildAt(0);
+ return Array.from({length: column.length}, (_, i) => {
+ if (typeof column.isValid === 'function' && !column.isValid(i)) {
+ return null;
+ }
+ return jsArrowCell(column.get(i));
+ });
+}
+
+test('compactArrowTable -> concat does not merge chunks; Vector has no combineChunks', t => {
+ const batchA = arrow.tableFromJSON([{lng: -122.4}]);
+ const batchB = arrow.tableFromJSON([{lng: -122.5}]);
+ const combined = batchA.concat(batchB);
+ const column = combined.getChildAt(0);
+
+ t.equal(
+ typeof column.combineChunks,
+ 'undefined',
+ 'apache-arrow JS Vector has concat, not combineChunks'
+ );
+ t.ok(column.data.length > 1, 'Table.concat keeps one chunk per record batch');
+ t.equal(typeof column.concat, 'function', 'Vector.concat exists but does not collapse chunks');
+
+ t.end();
+});
+
+test('compactArrowTable -> dictionary, WKB, and nested GeoArrow keep values', t => {
+ const geoMeta = metadata => {
+ const map = new Map();
+ map.set(GEOARROW_METADATA_KEY, metadata);
+ return map;
+ };
+
+ const dictType = new arrow.Dictionary(new arrow.Utf8(), new arrow.Int8());
+ const dictTable = concatColumnTables('city', dictType, ['sf', null], ['nyc']);
+ const dictCompacted = compactArrowTable(dictTable, 1);
+ t.equal(dictCompacted.batches.length, 1, 'dictionary column should collapse to one batch');
+ t.deepEqual(
+ columnCells(dictCompacted),
+ ['sf', null, 'nyc'],
+ 'dictionary values and nulls should survive the Builder rebuild'
+ );
+
+ const wkbPoint = (lng, lat) => {
+ const bytes = new Uint8Array(21);
+ const view = new DataView(bytes.buffer);
+ view.setUint8(0, 1);
+ view.setUint32(1, 1, true);
+ view.setFloat64(5, lng, true);
+ view.setFloat64(13, lat, true);
+ return bytes;
+ };
+ const wkbTable = concatColumnTables(
+ 'geometry',
+ new arrow.Binary(),
+ [wkbPoint(-122.4, 37.8), null],
+ [wkbPoint(-122.5, 37.9)],
+ geoMeta(GEOARROW_EXTENSIONS.WKB)
+ );
+ wkbTable.schema.metadata.set(
+ 'geo',
+ JSON.stringify({columns: {geometry: {encoding: 'WKB'}}})
+ );
+ const wkbCompacted = compactArrowTable(wkbTable, 1);
+ t.equal(wkbCompacted.batches.length, 1, 'WKB binary column should collapse to one batch');
+ t.deepEqual(
+ columnCells(wkbCompacted),
+ columnCells(wkbTable),
+ 'WKB bytes and nulls should survive the Builder rebuild'
+ );
+ t.equal(
+ wkbCompacted.schema.fields[0].metadata.get(GEOARROW_METADATA_KEY),
+ GEOARROW_EXTENSIONS.WKB,
+ 'geoarrow.wkb field metadata should be copied'
+ );
+ t.ok(wkbCompacted.schema.metadata.get('geo'), 'GeoParquet schema metadata should be copied');
+
+ const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false));
+ const pointTable = concatColumnTables(
+ 'geom',
+ pointType,
+ [[-122.4, 37.8], null],
+ [[-122.5, 37.9]],
+ geoMeta(GEOARROW_EXTENSIONS.POINT)
+ );
+ const pointCompacted = compactArrowTable(pointTable, 1);
+ t.equal(pointCompacted.batches.length, 1, 'geoarrow.point should collapse to one batch');
+ t.deepEqual(
+ columnCells(pointCompacted),
+ [[-122.4, 37.8], null, [-122.5, 37.9]],
+ 'geoarrow.point coordinates must not become NaN after Builder rebuild'
+ );
+
+ const lineType = new arrow.List(new arrow.Field('vertices', pointType, false));
+ const lineTable = concatColumnTables(
+ 'geom',
+ lineType,
+ [[[-122.4, 37.8], [-122.5, 37.9]]],
+ [[[0, 0], [1, 1]]],
+ geoMeta(GEOARROW_EXTENSIONS.LINESTRING)
+ );
+ const lineCompacted = compactArrowTable(lineTable, 1);
+ t.equal(lineCompacted.batches.length, 1, 'geoarrow.linestring should collapse to one batch');
+ t.deepEqual(
+ columnCells(lineCompacted),
+ columnCells(lineTable),
+ 'nested geoarrow.linestring coordinates should survive the Builder rebuild'
+ );
+
+ const polygonType = new arrow.List(new arrow.Field('rings', lineType, false));
+ const ring = [
+ [0, 0],
+ [1, 0],
+ [1, 1],
+ [0, 0]
+ ];
+ const polygonTable = concatColumnTables(
+ 'geom',
+ polygonType,
+ [[ring]],
+ [[ring]],
+ geoMeta(GEOARROW_EXTENSIONS.POLYGON)
+ );
+ const polygonCompacted = compactArrowTable(polygonTable, 1);
+ t.equal(polygonCompacted.batches.length, 1, 'geoarrow.polygon should collapse to one batch');
+ t.deepEqual(
+ columnCells(polygonCompacted),
+ columnCells(polygonTable),
+ 'nested geoarrow.polygon coordinates should survive the Builder rebuild'
+ );
+
+ t.end();
+});
+
+test('compactArrowTable -> duplicate field names keep both columns', t => {
+ const duplicateNameTable = (floatValues, intValues) => {
+ const schema = new arrow.Schema([
+ new arrow.Field('x', new arrow.Float64()),
+ new arrow.Field('x', new arrow.Int32())
+ ]);
+ const floats = arrow.vectorFromArray(floatValues, new arrow.Float64());
+ const ints = arrow.vectorFromArray(intValues, new arrow.Int32());
+ const data = arrow.makeData({
+ type: new arrow.Struct(schema.fields),
+ children: [floats.data[0], ints.data[0]],
+ length: floatValues.length
+ });
+ return new arrow.Table(new arrow.RecordBatch(schema, data));
+ };
+
+ const combined = duplicateNameTable([1.5], [1]).concat(duplicateNameTable([2.5], [2]));
+ t.equal(combined.numCols, 2, 'fixture should expose both duplicate-named columns');
+ t.ok(combined.batches.length > 1, 'fixture should have multiple record batches');
+
+ const compacted = compactArrowTable(combined, 1);
+ t.equal(compacted.numCols, 2, 'compaction should not drop a duplicate-named column');
+ t.equal(compacted.numRows, 2, 'compaction should keep all rows');
+ t.equal(compacted.batches.length, 1, 'duplicate-named columns should still collapse to one batch');
+ t.deepEqual(
+ [compacted.getChildAt(0).get(0), compacted.getChildAt(0).get(1)],
+ [1.5, 2.5],
+ 'first duplicate-named column should keep its float values'
+ );
+ t.deepEqual(
+ [compacted.getChildAt(1).get(0), compacted.getChildAt(1).get(1)],
+ [1, 2],
+ 'second duplicate-named column should keep its int values'
+ );
+
+ t.end();
+});
+
+test('compactArrowTable -> Builder failure keeps the original table', t => {
+ const batchA = arrow.tableFromJSON([{lng: -122.4}]);
+ const batchB = arrow.tableFromJSON([{lng: -122.5}]);
+ const combined = batchA.concat(batchB);
+ const exploding = {
+ batches: combined.batches,
+ numCols: combined.numCols,
+ schema: combined.schema,
+ getChildAt() {
+ throw new Error('cannot compact');
+ }
+ };
+
+ const result = compactArrowTable(exploding, 1);
+ t.equal(
+ result,
+ exploding,
+ 'compactArrowTable should return the original table when the Builder path throws'
+ );
+
+ t.end();
+});
diff --git a/test/node/utils/filter-utils-test.js b/test/node/utils/filter-utils-test.js
index 8ef54b5bb..41078920f 100644
--- a/test/node/utils/filter-utils-test.js
+++ b/test/node/utils/filter-utils-test.js
@@ -804,3 +804,117 @@ test('filterUtils -> getPolygonFilterFunctor -> aggregation layers (grid/hexagon
t.end();
});
+
+test('filterUtils -> getPolygonFilterFunctor -> point layer ignores non-finite altitude', t => {
+ const squarePolygon = {
+ type: 'Feature',
+ properties: {},
+ geometry: {
+ type: 'Polygon',
+ coordinates: [
+ [
+ [-1, -1],
+ [1, -1],
+ [1, 1],
+ [-1, 1],
+ [-1, -1]
+ ]
+ ]
+ }
+ };
+
+ const filter = {value: squarePolygon};
+ const layer = {
+ type: 'point',
+ config: {columnMode: 'points'},
+ getPositionAccessor: () => d => d.position,
+ dataToFeature: []
+ };
+
+ const fn = getPolygonFilterFunctor(layer, filter, null);
+
+ t.equal(fn({position: [0.5, 0.5, 0]}), true, 'finite altitude should keep a point inside');
+ t.equal(
+ fn({position: [0.5, 0.5, null]}),
+ true,
+ 'null altitude should not exclude a valid lng/lat'
+ );
+ t.equal(
+ fn({position: [0.5, 0.5, undefined]}),
+ true,
+ 'undefined altitude should not exclude a valid lng/lat'
+ );
+ t.equal(
+ fn({position: [10, 10, null]}),
+ false,
+ 'null altitude should not include a point outside'
+ );
+ t.equal(
+ fn({position: new Float64Array([0.5, 0.5, 0])}),
+ true,
+ 'typed-array lng/lat should be treated as a valid position'
+ );
+ t.equal(
+ fn({position: [null, 0.5, 0]}),
+ false,
+ 'null longitude should not coerce to 0 and pass the polygon filter'
+ );
+ t.equal(
+ fn({position: ['', 0.5, 0]}),
+ false,
+ 'empty-string longitude should not coerce to 0 and pass the polygon filter'
+ );
+
+ t.end();
+});
+
+test('filterUtils -> getPolygonFilterFunctor -> arc layer validates both endpoints', t => {
+ const squarePolygon = {
+ type: 'Feature',
+ properties: {},
+ geometry: {
+ type: 'Polygon',
+ coordinates: [
+ [
+ [-1, -1],
+ [1, -1],
+ [1, 1],
+ [-1, 1],
+ [-1, -1]
+ ]
+ ]
+ }
+ };
+
+ const fn = getPolygonFilterFunctor(
+ {
+ type: 'arc',
+ getPositionAccessor: () => d => d.position
+ },
+ {value: squarePolygon},
+ null
+ );
+
+ t.equal(
+ fn({position: [0.5, 0.5, 0, 0.2, 0.2, 0]}),
+ true,
+ 'arc with both endpoints inside should pass'
+ );
+ t.equal(
+ fn({position: [0.5, 0.5, null, 0.2, 0.2, undefined]}),
+ true,
+ 'null altitude should not exclude a valid arc'
+ );
+ t.equal(
+ fn({position: [0.5, 0.5, 0, null, 0.2, 0]}),
+ false,
+ 'missing destination longitude should not reach turfPoint'
+ );
+ t.equal(
+ fn({position: [0.5, 0.5, 0, 10, 10, 0]}),
+ false,
+ 'arc with destination outside should fail'
+ );
+
+ t.end();
+});