Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix: Allow passing arrow tables to ArrowDataContainer (#3247)

Ilya Boyandin2288bc3247e6
diff --git a/src/deckgl-arrow-layers/package.json b/src/deckgl-arrow-layers/package.json
index f4563820a..76e2b43fa 100644
--- a/src/deckgl-arrow-layers/package.json
+++ b/src/deckgl-arrow-layers/package.json
@@ -32,6 +32,7 @@
"dependencies": {
"@geoarrow/geoarrow-js": "^0.3.0",
"@kepler.gl/constants": "^3.2.2",
+ "@kepler.gl/utils": "^3.2.2",
"@math.gl/core": "^4.0.0",
"@math.gl/polygon": "^4.0.0",
"@math.gl/types": "^4.0.0",
diff --git a/src/deckgl-arrow-layers/src/utils/utils.ts b/src/deckgl-arrow-layers/src/utils/utils.ts
index a679fde4c..889e695c7 100644
--- a/src/deckgl-arrow-layers/src/utils/utils.ts
+++ b/src/deckgl-arrow-layers/src/utils/utils.ts
@@ -9,6 +9,7 @@ import {assert} from '@deck.gl/core/typed';
import * as arrow from 'apache-arrow';
import * as ga from '@geoarrow/geoarrow-js';
import {AccessorContext, AccessorFunction, _InternalAccessorContext} from '../types';
+import {isArrowFixedSizeList, isArrowStruct, isArrowVector} from '@kepler.gl/utils';
export type TypedArray =
| Uint8Array
@@ -45,14 +46,14 @@ function isDataInterleavedCoords(
data: arrow.Data
): data is arrow.Data<arrow.FixedSizeList<arrow.Float64>> {
// TODO: also check 2 or 3d? Float64?
- return data.type instanceof arrow.FixedSizeList;
+ return isArrowFixedSizeList(data.type);
}
function isDataSeparatedCoords(
data: arrow.Data
): data is arrow.Data<arrow.Struct<{x: arrow.Float64; y: arrow.Float64}>> {
// TODO: also check child names? Float64?
- return data.type instanceof arrow.Struct;
+ return isArrowStruct(data.type);
}
/**
@@ -162,7 +163,7 @@ export function assignAccessor(args: AssignAccessorProps) {
return;
}
- if (propInput instanceof arrow.Vector) {
+ if (isArrowVector(propInput)) {
const columnData = propInput.data[chunkIdx];
if (arrow.DataType.isFixedSizeList(columnData)) {
diff --git a/src/deckgl-arrow-layers/src/utils/validate.ts b/src/deckgl-arrow-layers/src/utils/validate.ts
index e9f52972c..af29bb325 100644
--- a/src/deckgl-arrow-layers/src/utils/validate.ts
+++ b/src/deckgl-arrow-layers/src/utils/validate.ts
@@ -6,6 +6,7 @@
// Copyright (c) vis.gl contributors
import {assert} from '@deck.gl/core/typed';
+import {isArrowVector} from '@kepler.gl/utils';
import * as arrow from 'apache-arrow';
export function validateAccessors(props: Record<string, any>, table: arrow.Table): void {
@@ -15,7 +16,7 @@ export function validateAccessors(props: Record<string, any>, table: arrow.Table
// Is it an accessor
if (accessorName.startsWith('get')) {
// Is it a vector accessor
- if (accessorValue instanceof arrow.Vector) {
+ if (isArrowVector(accessorValue)) {
vectorAccessors.push(accessorValue);
// Is it a color vector accessor
diff --git a/src/duckdb/package.json b/src/duckdb/package.json
index 362e0e95d..760bfccdf 100644
--- a/src/duckdb/package.json
+++ b/src/duckdb/package.json
@@ -31,6 +31,7 @@
"@kepler.gl/processors": "3.2.2",
"@kepler.gl/table": "3.2.2",
"@kepler.gl/types": "3.2.2",
+ "@kepler.gl/utils": "3.2.2",
"@monaco-editor/react": "^4.6.0",
"@radix-ui/react-collapsible": "^1.1.0",
"apache-arrow": ">=15.0.0",
diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.ts
index 9e112d0d6..aeee75b7e 100644
--- a/src/duckdb/src/table/duckdb-table.ts
+++ b/src/duckdb/src/table/duckdb-table.ts
@@ -19,7 +19,13 @@ import {
} from '@kepler.gl/processors';
import {KeplerTable} from '@kepler.gl/table';
import {Field} from '@kepler.gl/types';
-import {getApplicationConfig, DatabaseAdapter, DatabaseConnection} from '@kepler.gl/utils';
+import {
+ getApplicationConfig,
+ DatabaseAdapter,
+ DatabaseConnection,
+ isArrowTable,
+ isArrowVector
+} from '@kepler.gl/utils';
import {
processCsvRowObject,
@@ -161,10 +167,9 @@ export class KeplerGlDuckDbTable extends KeplerTable {
try {
// 1) data.rows contains an arrow table created by Add to Map data from DuckDb query.
// 2) arrow table is in cols & fields when a file is dragged & dropped into Add Data To Map dialog.
- const arrowTable =
- data.rows instanceof arrow.Table
- ? data.rows
- : restoreArrowTable(data.cols || [], data.fields, data.arrowSchema);
+ const arrowTable = isArrowTable(data.rows)
+ ? data.rows
+ : restoreArrowTable(data.cols || [], data.fields, data.arrowSchema);
// remove unsupported extensions from an arrow table that throw exceptions in DuckDB.
adjustedMetadata = removeUnsupportedExtensions(arrowTable);
@@ -215,7 +220,7 @@ export class KeplerGlDuckDbTable extends KeplerTable {
format = DATASET_FORMATS.row;
} else if (data.rows?.type === 'FeatureCollection') {
format = DATASET_FORMATS.geojson;
- } else if (data.cols?.[0] instanceof arrow.Vector) {
+ } else if (isArrowVector(data.cols?.[0])) {
format = DATASET_FORMATS.arrow;
}
}
diff --git a/src/layers/src/base-layer.ts b/src/layers/src/base-layer.ts
index 9b8324ab3..57f09446c 100644
--- a/src/layers/src/base-layer.ts
+++ b/src/layers/src/base-layer.ts
@@ -41,7 +41,8 @@ import {
hexToRgb,
isPlainObject,
isDomainStops,
- updateColorRangeByMatchingPalette
+ updateColorRangeByMatchingPalette,
+ isArrowTable
} from '@kepler.gl/utils';
import {generateHashId, toArray, notNullorUndefined} from '@kepler.gl/common-utils';
import {Datasets, GpuFilter, KeplerTable} from '@kepler.gl/table';
@@ -1585,7 +1586,7 @@ class Layer implements KeplerLayer {
const {data, mapState} = renderOpts;
const {textLabel} = this.config;
- const TextLayerClass = data.data instanceof arrow.Table ? GeoArrowTextLayer : TextLayer;
+ const TextLayerClass = isArrowTable(data.data) ? GeoArrowTextLayer : TextLayer;
return data.textLabels.reduce((accu, d, i) => {
if (d.getText) {
diff --git a/src/layers/src/layer-text-label.ts b/src/layers/src/layer-text-label.ts
index ebbeee7b6..a3838b72b 100644
--- a/src/layers/src/layer-text-label.ts
+++ b/src/layers/src/layer-text-label.ts
@@ -3,7 +3,7 @@
import * as arrow from 'apache-arrow';
import {getDistanceScales} from 'viewport-mercator-project';
-import {DataContainerInterface, ArrowDataContainer} from '@kepler.gl/utils';
+import {DataContainerInterface, ArrowDataContainer, isArrowTable} from '@kepler.gl/utils';
import {notNullorUndefined} from '@kepler.gl/common-utils';
import uniq from 'lodash/uniq';
@@ -82,7 +82,7 @@ export const formatTextLabelData = ({
getText = oldLayerData.textLabels[i].getText;
rebuildArrowTextVector = false;
} else {
- if (data instanceof arrow.Table) {
+ if (isArrowTable(data)) {
// we don't filter out arrow tables,
// so we use filteredIndex array instead
const allLabels: string[] = [];
@@ -108,7 +108,7 @@ export const formatTextLabelData = ({
// For now check here for ArrowTable, not ArrowDataContainer.
if (
rebuildArrowTextVector &&
- data instanceof arrow.Table &&
+ isArrowTable(data) &&
dataContainer instanceof ArrowDataContainer
) {
getText = dataContainer.getColumn(tl.field.fieldIdx);
diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.ts
index b54bdc5db..54dc89cae 100644
--- a/src/processors/src/file-handler.ts
+++ b/src/processors/src/file-handler.ts
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import * as arrow from 'apache-arrow';
import {parseInBatches} from '@loaders.gl/core';
import {JSONLoader, _JSONPath} from '@loaders.gl/json';
import {CSVLoader} from '@loaders.gl/csv';
@@ -12,7 +11,8 @@ import {
isPlainObject,
generateHashIdFromString,
getApplicationConfig,
- getError
+ getError,
+ isArrowTable
} from '@kepler.gl/utils';
import {generateHashId} from '@kepler.gl/common-utils';
import {DATASET_FORMATS} from '@kepler.gl/constants';
@@ -68,14 +68,7 @@ export type ProcessFileDataContent = {
metadata?: Map<string, string>;
};
-/**
- * check if table is an ArrowTable object
- * @param table - object to check
- * @returns {boolean} - true if table is an ArrowTable object type guarded
- */
-export function isArrowTable(table: any): table is arrow.Table {
- return Boolean(table instanceof arrow.Table);
-}
+export {isArrowTable};
/**
* check if data is an ArrowData object, which is an array of RecordBatch
diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.ts
index fe3154660..9ffde6a3e 100644
--- a/src/table/src/kepler-table.ts
+++ b/src/table/src/kepler-table.ts
@@ -241,7 +241,7 @@ class KeplerTable<F extends Field = Field> {
* @param data - new data e.g. the arrow data with new batches loaded
*/
async update(data: ProtoDataset['data']) {
- const dataContainerData = data.cols ? data.cols : data.rows;
+ const dataContainerData = data.arrowTable ?? data.cols ?? data.rows;
this.dataContainer.update?.(dataContainerData);
this.allIndexes = this.dataContainer.getPlainIndex();
this.filteredIndex = this.allIndexes;
diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.ts
index e32eb72f1..f462b11db 100644
--- a/src/utils/src/arrow-data-container.ts
+++ b/src/utils/src/arrow-data-container.ts
@@ -1,14 +1,14 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import { ALL_FIELD_TYPES } from '@kepler.gl/constants';
-import { ProtoDatasetField } from '@kepler.gl/types';
+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
+import {ProtoDatasetField} from '@kepler.gl/types';
import * as arrow from 'apache-arrow';
-import { console as globalConsole } from 'global/window';
-import { DATA_TYPES as AnalyzerDATA_TYPES } from 'type-analyzer';
+import {console as globalConsole} from 'global/window';
+import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer';
-import { DataContainerInterface, RangeOptions } from './data-container-interface';
-import { DataRow, SharedRowOptions } from './data-row';
+import {DataContainerInterface, RangeOptions} from './data-container-interface';
+import {DataRow, SharedRowOptions} from './data-row';
type ArrowDataContainerInput = {
cols: arrow.Vector[];
@@ -16,6 +16,87 @@ type ArrowDataContainerInput = {
arrowTable?: arrow.Table;
};
+/**
+ * Check if table is an ArrowTable object.
+ *
+ * We use duck-typing instead of `instanceof arrow.Table` because DuckDB loads its own
+ * bundled version of Apache Arrow. When DuckDB creates Arrow tables, they are instances
+ * of DuckDB's Arrow.Table class, not the Arrow.Table class from our application's
+ * apache-arrow package. This causes `instanceof` checks to fail even though the objects
+ * are functionally equivalent Arrow tables.
+ *
+ * @param data - object to check
+ * @returns true if data is an ArrowTable object (type guarded)
+ */
+export function isArrowTable(data: any): data is arrow.Table {
+ return (
+ typeof data === 'object' &&
+ data !== null &&
+ 'schema' in data &&
+ 'getChildAt' in data &&
+ typeof data.getChildAt === 'function' &&
+ 'batches' in data &&
+ Array.isArray(data.batches)
+ );
+}
+
+/**
+ * Check if data is an ArrowVector object.
+ * Uses duck-typing instead of `instanceof` to handle DuckDB's bundled Arrow version.
+ *
+ * @param data - object to check
+ * @returns true if data is an ArrowVector object (type guarded)
+ */
+export function isArrowVector(data: any): data is arrow.Vector {
+ return (
+ typeof data === 'object' &&
+ data !== null &&
+ 'type' in data &&
+ 'length' in data &&
+ typeof data.length === 'number' &&
+ 'get' in data &&
+ typeof data.get === 'function' &&
+ 'data' in data &&
+ Array.isArray(data.data)
+ );
+}
+
+/**
+ * Check if data is an Arrow FixedSizeList DataType.
+ * Uses duck-typing instead of `instanceof` to handle DuckDB's bundled Arrow version.
+ *
+ * @param data - object to check
+ * @returns true if data is an Arrow FixedSizeList DataType (type guarded)
+ */
+export function isArrowFixedSizeList(data: any): data is arrow.FixedSizeList {
+ return (
+ typeof data === 'object' &&
+ data !== null &&
+ 'typeId' in data &&
+ 'listSize' in data &&
+ typeof data.listSize === 'number' &&
+ 'children' in data &&
+ Array.isArray(data.children)
+ );
+}
+
+/**
+ * Check if data is an Arrow Struct DataType.
+ * Uses duck-typing instead of `instanceof` to handle DuckDB's bundled Arrow version.
+ *
+ * @param data - object to check
+ * @returns true if data is an Arrow Struct DataType (type guarded)
+ */
+export function isArrowStruct(data: any): data is arrow.Struct {
+ return (
+ typeof data === 'object' &&
+ data !== null &&
+ 'typeId' in data &&
+ 'children' in data &&
+ Array.isArray(data.children) &&
+ !('listSize' in data)
+ );
+}
/**
* @param dataContainer
@@ -89,13 +170,20 @@ export class ArrowDataContainer implements DataContainerInterface {
return this._arrowTable;
}
- update(updateData: arrow.Vector<any>[]) {
- this._cols = updateData;
- this._numColumns = this._cols.length;
- this._numRows = this._cols[0].length;
- this._numChunks = this._cols[0].data.length;
-
- this._arrowTable = this._createTable();
+ update(updateData: arrow.Vector<any>[] | arrow.Table) {
+ const isArrow = isArrowTable(updateData);
+ if (isArrow) {
+ this._cols = Array.from(
+ {length: updateData.numCols},
+ (_, i) => updateData.getChildAt(i) as arrow.Vector
+ ).filter(col => col);
+ } else {
+ this._cols = updateData;
+ }
+ this._numColumns = this._cols?.length ?? 0;
+ this._numRows = this._cols?.[0]?.length ?? 0;
+ this._numChunks = this._cols?.[0]?.data?.length ?? 0;
+ this._arrowTable = isArrow ? updateData : this._createTable();
// cache column data to make valueAt() faster
// this._colData = this._cols.map(c => c.toArray());
diff --git a/src/utils/src/data-container-interface.ts b/src/utils/src/data-container-interface.ts
index f785eb350..dd7b06bd1 100644
--- a/src/utils/src/data-container-interface.ts
+++ b/src/utils/src/data-container-interface.ts
@@ -3,6 +3,7 @@
import {ProtoDatasetField} from '@kepler.gl/types';
import {DataRow, SharedRowOptions} from './data-row';
+import * as arrow from 'apache-arrow';
/**
* Specifies a range of rows of a data container that should be processed.
@@ -14,9 +15,9 @@ export type RangeOptions = {
export interface DataContainerInterface {
/**
* Updates the data container with new data.
- * @param updateData updated data, e.g. for arrow data container, it's an array of arrow columns; for row data container, it's an array of rows.
+ * @param updateData updated data, e.g. for arrow data container, it's an array of arrow columns or an arrow.Table; for row data container, it's an array of rows.
*/
- update?(updateData: any[]): void;
+ update?(updateData: any[] | arrow.Table): void;
/**
* Returns the number of rows in the data container.
diff --git a/src/utils/src/data-utils.ts b/src/utils/src/data-utils.ts
index 93ff01ead..6ea05d57d 100644
--- a/src/utils/src/data-utils.ts
+++ b/src/utils/src/data-utils.ts
@@ -20,6 +20,7 @@ import {Field, Millisecond, ProtoDatasetField} from '@kepler.gl/types';
import {snapToMarks} from './plot';
import {isPlainObject} from './utils';
+import {isArrowVector} from './arrow-data-container';
export type FieldFormatter = (value: any, field?: ProtoDatasetField) => string;
@@ -317,7 +318,7 @@ export const FIELD_DISPLAY_FORMAT: {
? `[${String(d)}]`
: '',
[ALL_FIELD_TYPES.geoarrow]: (data, field) => {
- if (data instanceof arrow.Vector) {
+ if (isArrowVector(data)) {
try {
const encoding = field?.metadata?.get('ARROW:extension:name');
if (encoding) {
diff --git a/src/utils/src/index.ts b/src/utils/src/index.ts
index e33e65be3..f36b0ad8b 100644
--- a/src/utils/src/index.ts
+++ b/src/utils/src/index.ts
@@ -127,7 +127,11 @@ export * from './map-utils';
export {
ArrowDataContainer,
arrowDataTypeToAnalyzerDataType,
- arrowDataTypeToFieldType
+ arrowDataTypeToFieldType,
+ isArrowTable,
+ isArrowFixedSizeList,
+ isArrowStruct,
+ isArrowVector
} from './arrow-data-container';
export type {DataContainerInterface} from './data-container-interface';
export {