Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix: Allow passing arrow tables to ArrowDataContainer (#3247)
Ilya Boyandin2288bc3247e6
diff --git a/src/deckgl-arrow-layers/package.json b/src/deckgl-arrow-layers/package.jsonindex f4563820a..76e2b43fa 100644--- a/src/deckgl-arrow-layers/package.json+++ b/src/deckgl-arrow-layers/package.json@@ -32,6 +32,7 @@"dependencies": {"@geoarrow/geoarrow-js": "^0.3.0","@kepler.gl/constants": "^3.2.2",+ "@kepler.gl/utils": "^3.2.2","@math.gl/core": "^4.0.0","@math.gl/polygon": "^4.0.0","@math.gl/types": "^4.0.0",diff --git a/src/deckgl-arrow-layers/src/utils/utils.ts b/src/deckgl-arrow-layers/src/utils/utils.tsindex a679fde4c..889e695c7 100644--- a/src/deckgl-arrow-layers/src/utils/utils.ts+++ b/src/deckgl-arrow-layers/src/utils/utils.ts@@ -9,6 +9,7 @@ import {assert} from '@deck.gl/core/typed';import * as arrow from 'apache-arrow';import * as ga from '@geoarrow/geoarrow-js';import {AccessorContext, AccessorFunction, _InternalAccessorContext} from '../types';+import {isArrowFixedSizeList, isArrowStruct, isArrowVector} from '@kepler.gl/utils';export type TypedArray =| Uint8Array@@ -45,14 +46,14 @@ function isDataInterleavedCoords(data: arrow.Data): data is arrow.Data<arrow.FixedSizeList<arrow.Float64>> {// TODO: also check 2 or 3d? Float64?- return data.type instanceof arrow.FixedSizeList;+ return isArrowFixedSizeList(data.type);}function isDataSeparatedCoords(data: arrow.Data): data is arrow.Data<arrow.Struct<{x: arrow.Float64; y: arrow.Float64}>> {// TODO: also check child names? Float64?- return data.type instanceof arrow.Struct;+ return isArrowStruct(data.type);}/**@@ -162,7 +163,7 @@ export function assignAccessor(args: AssignAccessorProps) {return;}- if (propInput instanceof arrow.Vector) {+ if (isArrowVector(propInput)) {const columnData = propInput.data[chunkIdx];if (arrow.DataType.isFixedSizeList(columnData)) {diff --git a/src/deckgl-arrow-layers/src/utils/validate.ts b/src/deckgl-arrow-layers/src/utils/validate.tsindex e9f52972c..af29bb325 100644--- a/src/deckgl-arrow-layers/src/utils/validate.ts+++ b/src/deckgl-arrow-layers/src/utils/validate.ts@@ -6,6 +6,7 @@// Copyright (c) vis.gl contributorsimport {assert} from '@deck.gl/core/typed';+import {isArrowVector} from '@kepler.gl/utils';import * as arrow from 'apache-arrow';export function validateAccessors(props: Record<string, any>, table: arrow.Table): void {@@ -15,7 +16,7 @@ export function validateAccessors(props: Record<string, any>, table: arrow.Table// Is it an accessorif (accessorName.startsWith('get')) {// Is it a vector accessor- if (accessorValue instanceof arrow.Vector) {+ if (isArrowVector(accessorValue)) {vectorAccessors.push(accessorValue);// Is it a color vector accessordiff --git a/src/duckdb/package.json b/src/duckdb/package.jsonindex 362e0e95d..760bfccdf 100644--- a/src/duckdb/package.json+++ b/src/duckdb/package.json@@ -31,6 +31,7 @@"@kepler.gl/processors": "3.2.2","@kepler.gl/table": "3.2.2","@kepler.gl/types": "3.2.2",+ "@kepler.gl/utils": "3.2.2","@monaco-editor/react": "^4.6.0","@radix-ui/react-collapsible": "^1.1.0","apache-arrow": ">=15.0.0",diff --git a/src/duckdb/src/table/duckdb-table.ts b/src/duckdb/src/table/duckdb-table.tsindex 9e112d0d6..aeee75b7e 100644--- a/src/duckdb/src/table/duckdb-table.ts+++ b/src/duckdb/src/table/duckdb-table.ts@@ -19,7 +19,13 @@ import {} from '@kepler.gl/processors';import {KeplerTable} from '@kepler.gl/table';import {Field} from '@kepler.gl/types';-import {getApplicationConfig, DatabaseAdapter, DatabaseConnection} from '@kepler.gl/utils';+import {+ getApplicationConfig,+ DatabaseAdapter,+ DatabaseConnection,+ isArrowTable,+ isArrowVector+} from '@kepler.gl/utils';import {processCsvRowObject,@@ -161,10 +167,9 @@ export class KeplerGlDuckDbTable extends KeplerTable {try {// 1) data.rows contains an arrow table created by Add to Map data from DuckDb query.// 2) arrow table is in cols & fields when a file is dragged & dropped into Add Data To Map dialog.- const arrowTable =- data.rows instanceof arrow.Table- ? data.rows- : restoreArrowTable(data.cols || [], data.fields, data.arrowSchema);+ const arrowTable = isArrowTable(data.rows)+ ? data.rows+ : restoreArrowTable(data.cols || [], data.fields, data.arrowSchema);// remove unsupported extensions from an arrow table that throw exceptions in DuckDB.adjustedMetadata = removeUnsupportedExtensions(arrowTable);@@ -215,7 +220,7 @@ export class KeplerGlDuckDbTable extends KeplerTable {format = DATASET_FORMATS.row;} else if (data.rows?.type === 'FeatureCollection') {format = DATASET_FORMATS.geojson;- } else if (data.cols?.[0] instanceof arrow.Vector) {+ } else if (isArrowVector(data.cols?.[0])) {format = DATASET_FORMATS.arrow;}}diff --git a/src/layers/src/base-layer.ts b/src/layers/src/base-layer.tsindex 9b8324ab3..57f09446c 100644--- a/src/layers/src/base-layer.ts+++ b/src/layers/src/base-layer.ts@@ -41,7 +41,8 @@ import {hexToRgb,isPlainObject,isDomainStops,- updateColorRangeByMatchingPalette+ updateColorRangeByMatchingPalette,+ isArrowTable} from '@kepler.gl/utils';import {generateHashId, toArray, notNullorUndefined} from '@kepler.gl/common-utils';import {Datasets, GpuFilter, KeplerTable} from '@kepler.gl/table';@@ -1585,7 +1586,7 @@ class Layer implements KeplerLayer {const {data, mapState} = renderOpts;const {textLabel} = this.config;- const TextLayerClass = data.data instanceof arrow.Table ? GeoArrowTextLayer : TextLayer;+ const TextLayerClass = isArrowTable(data.data) ? GeoArrowTextLayer : TextLayer;return data.textLabels.reduce((accu, d, i) => {if (d.getText) {diff --git a/src/layers/src/layer-text-label.ts b/src/layers/src/layer-text-label.tsindex ebbeee7b6..a3838b72b 100644--- a/src/layers/src/layer-text-label.ts+++ b/src/layers/src/layer-text-label.ts@@ -3,7 +3,7 @@import * as arrow from 'apache-arrow';import {getDistanceScales} from 'viewport-mercator-project';-import {DataContainerInterface, ArrowDataContainer} from '@kepler.gl/utils';+import {DataContainerInterface, ArrowDataContainer, isArrowTable} from '@kepler.gl/utils';import {notNullorUndefined} from '@kepler.gl/common-utils';import uniq from 'lodash/uniq';@@ -82,7 +82,7 @@ export const formatTextLabelData = ({getText = oldLayerData.textLabels[i].getText;rebuildArrowTextVector = false;} else {- if (data instanceof arrow.Table) {+ if (isArrowTable(data)) {// we don't filter out arrow tables,// so we use filteredIndex array insteadconst allLabels: string[] = [];@@ -108,7 +108,7 @@ export const formatTextLabelData = ({// For now check here for ArrowTable, not ArrowDataContainer.if (rebuildArrowTextVector &&- data instanceof arrow.Table &&+ isArrowTable(data) &&dataContainer instanceof ArrowDataContainer) {getText = dataContainer.getColumn(tl.field.fieldIdx);diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.tsindex b54bdc5db..54dc89cae 100644--- a/src/processors/src/file-handler.ts+++ b/src/processors/src/file-handler.ts@@ -1,7 +1,6 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import * as arrow from 'apache-arrow';import {parseInBatches} from '@loaders.gl/core';import {JSONLoader, _JSONPath} from '@loaders.gl/json';import {CSVLoader} from '@loaders.gl/csv';@@ -12,7 +11,8 @@ import {isPlainObject,generateHashIdFromString,getApplicationConfig,- getError+ getError,+ isArrowTable} from '@kepler.gl/utils';import {generateHashId} from '@kepler.gl/common-utils';import {DATASET_FORMATS} from '@kepler.gl/constants';@@ -68,14 +68,7 @@ export type ProcessFileDataContent = {metadata?: Map<string, string>;};-/**- * check if table is an ArrowTable object- * @param table - object to check- * @returns {boolean} - true if table is an ArrowTable object type guarded- */-export function isArrowTable(table: any): table is arrow.Table {- return Boolean(table instanceof arrow.Table);-}+export {isArrowTable};/*** check if data is an ArrowData object, which is an array of RecordBatchdiff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.tsindex fe3154660..9ffde6a3e 100644--- a/src/table/src/kepler-table.ts+++ b/src/table/src/kepler-table.ts@@ -241,7 +241,7 @@ class KeplerTable<F extends Field = Field> {* @param data - new data e.g. the arrow data with new batches loaded*/async update(data: ProtoDataset['data']) {- const dataContainerData = data.cols ? data.cols : data.rows;+ const dataContainerData = data.arrowTable ?? data.cols ?? data.rows;this.dataContainer.update?.(dataContainerData);this.allIndexes = this.dataContainer.getPlainIndex();this.filteredIndex = this.allIndexes;diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.tsindex e32eb72f1..f462b11db 100644--- a/src/utils/src/arrow-data-container.ts+++ b/src/utils/src/arrow-data-container.ts@@ -1,14 +1,14 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import { ALL_FIELD_TYPES } from '@kepler.gl/constants';-import { ProtoDatasetField } from '@kepler.gl/types';+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';+import {ProtoDatasetField} from '@kepler.gl/types';import * as arrow from 'apache-arrow';-import { console as globalConsole } from 'global/window';-import { DATA_TYPES as AnalyzerDATA_TYPES } from 'type-analyzer';+import {console as globalConsole} from 'global/window';+import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer';-import { DataContainerInterface, RangeOptions } from './data-container-interface';-import { DataRow, SharedRowOptions } from './data-row';+import {DataContainerInterface, RangeOptions} from './data-container-interface';+import {DataRow, SharedRowOptions} from './data-row';type ArrowDataContainerInput = {cols: arrow.Vector[];@@ -16,6 +16,87 @@ type ArrowDataContainerInput = {arrowTable?: arrow.Table;};+/**+ * Check if table is an ArrowTable object.+ *+ * We use duck-typing instead of `instanceof arrow.Table` because DuckDB loads its own+ * bundled version of Apache Arrow. When DuckDB creates Arrow tables, they are instances+ * of DuckDB's Arrow.Table class, not the Arrow.Table class from our application's+ * apache-arrow package. This causes `instanceof` checks to fail even though the objects+ * are functionally equivalent Arrow tables.+ *+ * @param data - object to check+ * @returns true if data is an ArrowTable object (type guarded)+ */+export function isArrowTable(data: any): data is arrow.Table {+ return (+ typeof data === 'object' &&+ data !== null &&+ 'schema' in data &&+ 'getChildAt' in data &&+ typeof data.getChildAt === 'function' &&+ 'batches' in data &&+ Array.isArray(data.batches)+ );+}++/**+ * Check if data is an ArrowVector object.+ * Uses duck-typing instead of `instanceof` to handle DuckDB's bundled Arrow version.+ *+ * @param data - object to check+ * @returns true if data is an ArrowVector object (type guarded)+ */+export function isArrowVector(data: any): data is arrow.Vector {+ return (+ typeof data === 'object' &&+ data !== null &&+ 'type' in data &&+ 'length' in data &&+ typeof data.length === 'number' &&+ 'get' in data &&+ typeof data.get === 'function' &&+ 'data' in data &&+ Array.isArray(data.data)+ );+}++/**+ * Check if data is an Arrow FixedSizeList DataType.+ * Uses duck-typing instead of `instanceof` to handle DuckDB's bundled Arrow version.+ *+ * @param data - object to check+ * @returns true if data is an Arrow FixedSizeList DataType (type guarded)+ */+export function isArrowFixedSizeList(data: any): data is arrow.FixedSizeList {+ return (+ typeof data === 'object' &&+ data !== null &&+ 'typeId' in data &&+ 'listSize' in data &&+ typeof data.listSize === 'number' &&+ 'children' in data &&+ Array.isArray(data.children)+ );+}++/**+ * Check if data is an Arrow Struct DataType.+ * Uses duck-typing instead of `instanceof` to handle DuckDB's bundled Arrow version.+ *+ * @param data - object to check+ * @returns true if data is an Arrow Struct DataType (type guarded)+ */+export function isArrowStruct(data: any): data is arrow.Struct {+ return (+ typeof data === 'object' &&+ data !== null &&+ 'typeId' in data &&+ 'children' in data &&+ Array.isArray(data.children) &&+ !('listSize' in data)+ );+}/*** @param dataContainer@@ -89,13 +170,20 @@ export class ArrowDataContainer implements DataContainerInterface {return this._arrowTable;}- update(updateData: arrow.Vector<any>[]) {- this._cols = updateData;- this._numColumns = this._cols.length;- this._numRows = this._cols[0].length;- this._numChunks = this._cols[0].data.length;-- this._arrowTable = this._createTable();+ update(updateData: arrow.Vector<any>[] | arrow.Table) {+ const isArrow = isArrowTable(updateData);+ if (isArrow) {+ this._cols = Array.from(+ {length: updateData.numCols},+ (_, i) => updateData.getChildAt(i) as arrow.Vector+ ).filter(col => col);+ } else {+ this._cols = updateData;+ }+ this._numColumns = this._cols?.length ?? 0;+ this._numRows = this._cols?.[0]?.length ?? 0;+ this._numChunks = this._cols?.[0]?.data?.length ?? 0;+ this._arrowTable = isArrow ? updateData : this._createTable();// cache column data to make valueAt() faster// this._colData = this._cols.map(c => c.toArray());diff --git a/src/utils/src/data-container-interface.ts b/src/utils/src/data-container-interface.tsindex f785eb350..dd7b06bd1 100644--- a/src/utils/src/data-container-interface.ts+++ b/src/utils/src/data-container-interface.ts@@ -3,6 +3,7 @@import {ProtoDatasetField} from '@kepler.gl/types';import {DataRow, SharedRowOptions} from './data-row';+import * as arrow from 'apache-arrow';/*** Specifies a range of rows of a data container that should be processed.@@ -14,9 +15,9 @@ export type RangeOptions = {export interface DataContainerInterface {/*** Updates the data container with new data.- * @param updateData updated data, e.g. for arrow data container, it's an array of arrow columns; for row data container, it's an array of rows.+ * @param updateData updated data, e.g. for arrow data container, it's an array of arrow columns or an arrow.Table; for row data container, it's an array of rows.*/- update?(updateData: any[]): void;+ update?(updateData: any[] | arrow.Table): void;/*** Returns the number of rows in the data container.diff --git a/src/utils/src/data-utils.ts b/src/utils/src/data-utils.tsindex 93ff01ead..6ea05d57d 100644--- a/src/utils/src/data-utils.ts+++ b/src/utils/src/data-utils.ts@@ -20,6 +20,7 @@ import {Field, Millisecond, ProtoDatasetField} from '@kepler.gl/types';import {snapToMarks} from './plot';import {isPlainObject} from './utils';+import {isArrowVector} from './arrow-data-container';export type FieldFormatter = (value: any, field?: ProtoDatasetField) => string;@@ -317,7 +318,7 @@ export const FIELD_DISPLAY_FORMAT: {? `[${String(d)}]`: '',[ALL_FIELD_TYPES.geoarrow]: (data, field) => {- if (data instanceof arrow.Vector) {+ if (isArrowVector(data)) {try {const encoding = field?.metadata?.get('ARROW:extension:name');if (encoding) {diff --git a/src/utils/src/index.ts b/src/utils/src/index.tsindex e33e65be3..f36b0ad8b 100644--- a/src/utils/src/index.ts+++ b/src/utils/src/index.ts@@ -127,7 +127,11 @@ export * from './map-utils';export {ArrowDataContainer,arrowDataTypeToAnalyzerDataType,- arrowDataTypeToFieldType+ arrowDataTypeToFieldType,+ isArrowTable,+ isArrowFixedSizeList,+ isArrowStruct,+ isArrowVector} from './arrow-data-container';export type {DataContainerInterface} from './data-container-interface';export {