Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix(exported-html): include only one copy of arrow (#3717)
Igor Dykhta862aa5cb068f
diff --git a/esbuild/umd-esbuild.config.mjs b/esbuild/umd-esbuild.config.mjsindex d2f9bceb8..14498780e 100644--- a/esbuild/umd-esbuild.config.mjs+++ b/esbuild/umd-esbuild.config.mjs@@ -123,6 +123,15 @@ const config = {// of the external `React` global by jsxRuntimeShimPlugin below.external: ['react', 'react-dom', 'redux', 'react-redux', 'styled-components'],+ // loaders.gl's ParquetArrowLoader calls convertArrowToSchema, which matches+ // column types with `switch (arrowType.constructor)`. Duplicate apache-arrow+ // copies (kepler 17 vs nested loaders.gl 21) make that identity check fail+ // and throw `arrow type not supported: tL` in the minified exported HTML.+ // Force a single copy, matching the demo-app / website bundler aliases.+ alias: {+ 'apache-arrow': join(NODE_MODULES_DIR, 'apache-arrow')+ },+plugins: [jsxRuntimeShimPlugin,audioRecorderStubPlugin,diff --git a/package.json b/package.jsonindex 5c137770d..63a4933dd 100644--- a/package.json+++ b/package.json@@ -275,6 +275,7 @@"@loaders.gl/wkt": "4.4.1","@loaders.gl/wms": "4.4.1","@loaders.gl/zip": "4.4.1",+ "apache-arrow": "17.0.0","@luma.gl/constants": "9.3.2","@luma.gl/core": "9.3.2","@luma.gl/effects": "9.3.2",diff --git a/test/node/processors/file-handler-test.js b/test/node/processors/file-handler-test.jsindex 18d57a54f..e357d5a28 100644--- a/test/node/processors/file-handler-test.js+++ b/test/node/processors/file-handler-test.js@@ -12,6 +12,7 @@ import {} from '@kepler.gl/processors';import {getDatasetRefreshIntervalMs} from '@kepler.gl/constants';import * as arrow from 'apache-arrow';+import {convertArrowToSchema} from '@loaders.gl/schema-utils';import {parsedFields, parsedRows} from 'test/fixtures/row-object';import {savedStateV1InteractionCoordinate as keplerglMap,@@ -349,6 +350,30 @@ test('#file-handler -> processFileData remote ids do not collide on filename', at.end();});+test('#file-handler -> convertArrowToSchema uses a single apache-arrow copy', t => {+ // Parquet loading serializes the Arrow schema via loaders.gl. If kepler.gl+ // and @loaders.gl/schema-utils resolve different apache-arrow packages, that+ // switch-on-constructor check throws `arrow type not supported: <Class>`+ // (minified to e.g. `tL` in exported HTML).+ const table = new arrow.Table({+ lat: arrow.vectorFromArray(new Float64Array([37.8])),+ lng: arrow.vectorFromArray(new Float64Array([-122.4])),+ name: arrow.vectorFromArray(['alpha'], new arrow.Utf8())+ });++ let schema;+ t.doesNotThrow(() => {+ schema = convertArrowToSchema(table.schema);+ }, 'schema conversion should not fail with a constructor identity mismatch');+ t.deepEqual(+ schema.fields.map(field => field.type),+ ['float64', 'float64', 'utf8'],+ 'should serialize primitive Arrow types from the shared apache-arrow copy'+ );++ t.end();+});+test('#file-handler -> processArrowBatches skip compact for incremental loads', t => {const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]);const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]);diff --git a/yarn.lock b/yarn.lockindex 8ef8f4090..f4ffdb3c8 100644--- a/yarn.lock+++ b/yarn.lock@@ -8581,15 +8581,6 @@ __metadata:languageName: nodelinkType: hard-"@types/node@npm:^24.0.3":- version: 24.12.2- resolution: "@types/node@npm:24.12.2"- dependencies:- undici-types: "npm:~7.16.0"- checksum: 10c0/710050c42f89075c4479e4e1e4c2532486b0c41b1e2a8a13ad88641c88b88cdaea87414e19224f30028719737bd70e327edcaa184d50e86b9418941edd7eb02b- languageName: node- linkType: hard-"@types/normalize-package-data@npm:^2.4.0":version: 2.4.4resolution: "@types/normalize-package-data@npm:2.4.4"@@ -9914,26 +9905,7 @@ __metadata:languageName: nodelinkType: hard-"apache-arrow@npm:>= 17.0.0":- version: 21.1.0- resolution: "apache-arrow@npm:21.1.0"- dependencies:- "@swc/helpers": "npm:^0.5.11"- "@types/command-line-args": "npm:^5.2.3"- "@types/command-line-usage": "npm:^5.0.4"- "@types/node": "npm:^24.0.3"- command-line-args: "npm:^6.0.1"- command-line-usage: "npm:^7.0.1"- flatbuffers: "npm:^25.1.24"- json-bignum: "npm:^0.0.3"- tslib: "npm:^2.6.2"- bin:- arrow2csv: bin/arrow2csv.js- checksum: 10c0/d689b433d0a07bf8741870bd9ae363a64efe73394f6b0a59eb0732eb33c388d657273a4e2a7f89827add7ad7dcb3680b0a9a34f6f1818b370b3b22b43e06d6e1- languageName: node- linkType: hard--"apache-arrow@npm:>=15, apache-arrow@npm:>=15.0.0, apache-arrow@npm:^17.0.0":+"apache-arrow@npm:17.0.0":version: 17.0.0resolution: "apache-arrow@npm:17.0.0"dependencies:@@ -10060,13 +10032,6 @@ __metadata:languageName: nodelinkType: hard-"array-back@npm:^6.2.3":- version: 6.2.3- resolution: "array-back@npm:6.2.3"- checksum: 10c0/921bd7857112b8fbe3bc72f3498ca7c1c1652406e966a2468dcb8f67672c43e8b58e734138141c22fc21c357e59eaccc44b7727738a666f1056fc1f646b335df- languageName: node- linkType: hard-"array-buffer-byte-length@npm:^1.0.0, array-buffer-byte-length@npm:^1.0.1":version: 1.0.1resolution: "array-buffer-byte-length@npm:1.0.1"@@ -12270,23 +12235,6 @@ __metadata:languageName: nodelinkType: hard-"command-line-args@npm:^6.0.1":- version: 6.0.2- resolution: "command-line-args@npm:6.0.2"- dependencies:- array-back: "npm:^6.2.3"- find-replace: "npm:^5.0.2"- lodash.camelcase: "npm:^4.3.0"- typical: "npm:^7.3.0"- peerDependencies:- "@75lb/nature": "*"- peerDependenciesMeta:- "@75lb/nature":- optional: true- checksum: 10c0/4d66a1c4ae86bd86d9934ae85410bdedbe8796a060ad4e7cf7b9ec8275044f114b30965436a2c9a91f86d7a4279fb1ae825019f5861ddf048f275c6a0f24aa02- languageName: node- linkType: hard-"command-line-usage@npm:^7.0.1":version: 7.0.3resolution: "command-line-usage@npm:7.0.3"@@ -16035,18 +15983,6 @@ __metadata:languageName: nodelinkType: hard-"find-replace@npm:^5.0.2":- version: 5.0.2- resolution: "find-replace@npm:5.0.2"- peerDependencies:- "@75lb/nature": "*"- peerDependenciesMeta:- "@75lb/nature":- optional: true- checksum: 10c0/25db7167e8767b0683251a985af82e29b0ac26003fe2cd6ddd86e4f2c83fc10d97a81c6f6686034d8c2b9ee88bc53547bf86cc103479a7323342da5ace9f6504- languageName: node- linkType: hard-"find-up@npm:^2.0.0":version: 2.1.0resolution: "find-up@npm:2.1.0"@@ -16112,13 +16048,6 @@ __metadata:languageName: nodelinkType: hard-"flatbuffers@npm:^25.1.24":- version: 25.9.23- resolution: "flatbuffers@npm:25.9.23"- checksum: 10c0/957c4ae2a02be1703c98b36b4dc8ceb81613cf8e2333026afc95a7b68b088bed5458056dc29d0ab7ce8bc403b8c003732b0968d24aba46f5e7c8f71789a6bd9e- languageName: node- linkType: hard-"flatted@npm:^3.2.9":version: 3.4.3resolution: "flatted@npm:3.4.3"@@ -29128,13 +29057,6 @@ __metadata:languageName: nodelinkType: hard-"typical@npm:^7.3.0":- version: 7.3.0- resolution: "typical@npm:7.3.0"- checksum: 10c0/bee697a88e1dd0447bc1cf7f6e875eaa2b0fb2cccb338b7b261e315f7df84a66402864bfc326d6b3117c50475afd1d49eda03d846a6299ad25f211035bfab3b1- languageName: node- linkType: hard-"uglify-js@npm:^3.1.4":version: 3.19.0resolution: "uglify-js@npm:3.19.0"@@ -29201,13 +29123,6 @@ __metadata:languageName: nodelinkType: hard-"undici-types@npm:~7.16.0":- version: 7.16.0- resolution: "undici-types@npm:7.16.0"- checksum: 10c0/3033e2f2b5c9f1504bdc5934646cb54e37ecaca0f9249c983f7b1fc2e87c6d18399ebb05dc7fd5419e02b2e915f734d872a65da2e3eeed1813951c427d33cc9a- languageName: node- linkType: hard-"unherit@npm:^1.0.4":version: 1.1.3resolution: "unherit@npm:1.1.3"