Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix(exported-html): include only one copy of arrow (#3717)

Igor Dykhta862aa5cb068f
diff --git a/esbuild/umd-esbuild.config.mjs b/esbuild/umd-esbuild.config.mjs
index d2f9bceb8..14498780e 100644
--- a/esbuild/umd-esbuild.config.mjs
+++ b/esbuild/umd-esbuild.config.mjs
@@ -123,6 +123,15 @@ const config = {
// of the external `React` global by jsxRuntimeShimPlugin below.
external: ['react', 'react-dom', 'redux', 'react-redux', 'styled-components'],
+ // loaders.gl's ParquetArrowLoader calls convertArrowToSchema, which matches
+ // column types with `switch (arrowType.constructor)`. Duplicate apache-arrow
+ // copies (kepler 17 vs nested loaders.gl 21) make that identity check fail
+ // and throw `arrow type not supported: tL` in the minified exported HTML.
+ // Force a single copy, matching the demo-app / website bundler aliases.
+ alias: {
+ 'apache-arrow': join(NODE_MODULES_DIR, 'apache-arrow')
+ },
+
plugins: [
jsxRuntimeShimPlugin,
audioRecorderStubPlugin,
diff --git a/package.json b/package.json
index 5c137770d..63a4933dd 100644
--- a/package.json
+++ b/package.json
@@ -275,6 +275,7 @@
"@loaders.gl/wkt": "4.4.1",
"@loaders.gl/wms": "4.4.1",
"@loaders.gl/zip": "4.4.1",
+ "apache-arrow": "17.0.0",
"@luma.gl/constants": "9.3.2",
"@luma.gl/core": "9.3.2",
"@luma.gl/effects": "9.3.2",
diff --git a/test/node/processors/file-handler-test.js b/test/node/processors/file-handler-test.js
index 18d57a54f..e357d5a28 100644
--- a/test/node/processors/file-handler-test.js
+++ b/test/node/processors/file-handler-test.js
@@ -12,6 +12,7 @@ import {
} from '@kepler.gl/processors';
import {getDatasetRefreshIntervalMs} from '@kepler.gl/constants';
import * as arrow from 'apache-arrow';
+import {convertArrowToSchema} from '@loaders.gl/schema-utils';
import {parsedFields, parsedRows} from 'test/fixtures/row-object';
import {
savedStateV1InteractionCoordinate as keplerglMap,
@@ -349,6 +350,30 @@ test('#file-handler -> processFileData remote ids do not collide on filename', a
t.end();
});
+test('#file-handler -> convertArrowToSchema uses a single apache-arrow copy', t => {
+ // Parquet loading serializes the Arrow schema via loaders.gl. If kepler.gl
+ // and @loaders.gl/schema-utils resolve different apache-arrow packages, that
+ // switch-on-constructor check throws `arrow type not supported: <Class>`
+ // (minified to e.g. `tL` in exported HTML).
+ const table = new arrow.Table({
+ lat: arrow.vectorFromArray(new Float64Array([37.8])),
+ lng: arrow.vectorFromArray(new Float64Array([-122.4])),
+ name: arrow.vectorFromArray(['alpha'], new arrow.Utf8())
+ });
+
+ let schema;
+ t.doesNotThrow(() => {
+ schema = convertArrowToSchema(table.schema);
+ }, 'schema conversion should not fail with a constructor identity mismatch');
+ t.deepEqual(
+ schema.fields.map(field => field.type),
+ ['float64', 'float64', 'utf8'],
+ 'should serialize primitive Arrow types from the shared apache-arrow copy'
+ );
+
+ t.end();
+});
+
test('#file-handler -> processArrowBatches skip compact for incremental loads', t => {
const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]);
const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]);
diff --git a/yarn.lock b/yarn.lock
index 8ef8f4090..f4ffdb3c8 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -8581,15 +8581,6 @@ __metadata:
languageName: node
linkType: hard
-"@types/node@npm:^24.0.3":
- version: 24.12.2
- resolution: "@types/node@npm:24.12.2"
- dependencies:
- undici-types: "npm:~7.16.0"
- checksum: 10c0/710050c42f89075c4479e4e1e4c2532486b0c41b1e2a8a13ad88641c88b88cdaea87414e19224f30028719737bd70e327edcaa184d50e86b9418941edd7eb02b
- languageName: node
- linkType: hard
-
"@types/normalize-package-data@npm:^2.4.0":
version: 2.4.4
resolution: "@types/normalize-package-data@npm:2.4.4"
@@ -9914,26 +9905,7 @@ __metadata:
languageName: node
linkType: hard
-"apache-arrow@npm:>= 17.0.0":
- version: 21.1.0
- resolution: "apache-arrow@npm:21.1.0"
- dependencies:
- "@swc/helpers": "npm:^0.5.11"
- "@types/command-line-args": "npm:^5.2.3"
- "@types/command-line-usage": "npm:^5.0.4"
- "@types/node": "npm:^24.0.3"
- command-line-args: "npm:^6.0.1"
- command-line-usage: "npm:^7.0.1"
- flatbuffers: "npm:^25.1.24"
- json-bignum: "npm:^0.0.3"
- tslib: "npm:^2.6.2"
- bin:
- arrow2csv: bin/arrow2csv.js
- checksum: 10c0/d689b433d0a07bf8741870bd9ae363a64efe73394f6b0a59eb0732eb33c388d657273a4e2a7f89827add7ad7dcb3680b0a9a34f6f1818b370b3b22b43e06d6e1
- languageName: node
- linkType: hard
-
-"apache-arrow@npm:>=15, apache-arrow@npm:>=15.0.0, apache-arrow@npm:^17.0.0":
+"apache-arrow@npm:17.0.0":
version: 17.0.0
resolution: "apache-arrow@npm:17.0.0"
dependencies:
@@ -10060,13 +10032,6 @@ __metadata:
languageName: node
linkType: hard
-"array-back@npm:^6.2.3":
- version: 6.2.3
- resolution: "array-back@npm:6.2.3"
- checksum: 10c0/921bd7857112b8fbe3bc72f3498ca7c1c1652406e966a2468dcb8f67672c43e8b58e734138141c22fc21c357e59eaccc44b7727738a666f1056fc1f646b335df
- languageName: node
- linkType: hard
-
"array-buffer-byte-length@npm:^1.0.0, array-buffer-byte-length@npm:^1.0.1":
version: 1.0.1
resolution: "array-buffer-byte-length@npm:1.0.1"
@@ -12270,23 +12235,6 @@ __metadata:
languageName: node
linkType: hard
-"command-line-args@npm:^6.0.1":
- version: 6.0.2
- resolution: "command-line-args@npm:6.0.2"
- dependencies:
- array-back: "npm:^6.2.3"
- find-replace: "npm:^5.0.2"
- lodash.camelcase: "npm:^4.3.0"
- typical: "npm:^7.3.0"
- peerDependencies:
- "@75lb/nature": "*"
- peerDependenciesMeta:
- "@75lb/nature":
- optional: true
- checksum: 10c0/4d66a1c4ae86bd86d9934ae85410bdedbe8796a060ad4e7cf7b9ec8275044f114b30965436a2c9a91f86d7a4279fb1ae825019f5861ddf048f275c6a0f24aa02
- languageName: node
- linkType: hard
-
"command-line-usage@npm:^7.0.1":
version: 7.0.3
resolution: "command-line-usage@npm:7.0.3"
@@ -16035,18 +15983,6 @@ __metadata:
languageName: node
linkType: hard
-"find-replace@npm:^5.0.2":
- version: 5.0.2
- resolution: "find-replace@npm:5.0.2"
- peerDependencies:
- "@75lb/nature": "*"
- peerDependenciesMeta:
- "@75lb/nature":
- optional: true
- checksum: 10c0/25db7167e8767b0683251a985af82e29b0ac26003fe2cd6ddd86e4f2c83fc10d97a81c6f6686034d8c2b9ee88bc53547bf86cc103479a7323342da5ace9f6504
- languageName: node
- linkType: hard
-
"find-up@npm:^2.0.0":
version: 2.1.0
resolution: "find-up@npm:2.1.0"
@@ -16112,13 +16048,6 @@ __metadata:
languageName: node
linkType: hard
-"flatbuffers@npm:^25.1.24":
- version: 25.9.23
- resolution: "flatbuffers@npm:25.9.23"
- checksum: 10c0/957c4ae2a02be1703c98b36b4dc8ceb81613cf8e2333026afc95a7b68b088bed5458056dc29d0ab7ce8bc403b8c003732b0968d24aba46f5e7c8f71789a6bd9e
- languageName: node
- linkType: hard
-
"flatted@npm:^3.2.9":
version: 3.4.3
resolution: "flatted@npm:3.4.3"
@@ -29128,13 +29057,6 @@ __metadata:
languageName: node
linkType: hard
-"typical@npm:^7.3.0":
- version: 7.3.0
- resolution: "typical@npm:7.3.0"
- checksum: 10c0/bee697a88e1dd0447bc1cf7f6e875eaa2b0fb2cccb338b7b261e315f7df84a66402864bfc326d6b3117c50475afd1d49eda03d846a6299ad25f211035bfab3b1
- languageName: node
- linkType: hard
-
"uglify-js@npm:^3.1.4":
version: 3.19.0
resolution: "uglify-js@npm:3.19.0"
@@ -29201,13 +29123,6 @@ __metadata:
languageName: node
linkType: hard
-"undici-types@npm:~7.16.0":
- version: 7.16.0
- resolution: "undici-types@npm:7.16.0"
- checksum: 10c0/3033e2f2b5c9f1504bdc5934646cb54e37ecaca0f9249c983f7b1fc2e87c6d18399ebb05dc7fd5419e02b2e915f734d872a65da2e3eeed1813951c427d33cc9a
- languageName: node
- linkType: hard
-
"unherit@npm:^1.0.4":
version: 1.1.3
resolution: "unherit@npm:1.1.3"