Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
feat: add async loaders.gl 4.4 loader registry (#3682)
Ib Green4fce8abdbb4f
diff --git a/src/layers/src/layer-utils.ts b/src/layers/src/layer-utils.tsindex ee074093e..30390abde 100644--- a/src/layers/src/layer-utils.ts+++ b/src/layers/src/layer-utils.ts@@ -21,6 +21,7 @@ import {getBinaryGeometriesFromArrow, BinaryGeometriesFromArrowOptions} from '@limport {updateBoundsFromGeoArrowSamples} from '@loaders.gl/geoarrow';import {convertGeoArrowGeometryToGeoJSON} from '@loaders.gl/gis';+import {parseSync} from '@loaders.gl/core';import {WKBLoader} from '@loaders.gl/wkt';import {geojsonToBinary} from '@loaders.gl/gis';import {@@ -112,7 +113,7 @@ function getBinaryGeometriesFromWKBArrow(if (chunk.valueOffsets[i + 1] - chunk.valueOffsets[i] > 0) {const valuesSlice = chunk.values.slice(chunk.valueOffsets[i], chunk.valueOffsets[i + 1]);- const geometry = WKBLoader?.parseSync?.(valuesSlice.buffer, {+ const geometry = parseSync(valuesSlice.buffer, WKBLoader, {wkb: {shape: 'geojson-geometry'}}) as Geometry;const feature: Feature = {diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.tsindex 9c48a148c..0fd1756aa 100644--- a/src/processors/src/file-handler.ts+++ b/src/processors/src/file-handler.ts@@ -2,10 +2,6 @@// Copyright contributors to the kepler.gl projectimport {parseInBatches} from '@loaders.gl/core';-import {JSONLoader, _JSONPath} from '@loaders.gl/json';-import {CSVLoader} from '@loaders.gl/csv';-import {GeoArrowLoader} from '@loaders.gl/arrow';-import {ParquetArrowLoader} from '@loaders.gl/parquet';import {Loader} from '@loaders.gl/loader-utils';import {isPlainObject,@@ -28,6 +24,7 @@ import {} from './data-processor';import {FileCacheItem, ValidKeplerGlMap} from './types';+import {getKeplerLoaders} from './loader-registry';const BATCH_TYPE = {METADATA: 'metadata',@@ -184,6 +181,7 @@ export async function* readBatch(// Set the streamed data correctly is Batch json path is set// and the path streamed is not the top level object (jsonpath = '$')if (batch.jsonpath && batch.jsonpath.length > 1) {+ const {_JSONPath} = await import('@loaders.gl/json');const streamingPath = new _JSONPath(batch.jsonpath);streamingPath.setFieldAtPath(result, batches);} else if (batch.jsonpath && batch.jsonpath.length === 1) {@@ -231,7 +229,7 @@ export async function readFileInBatches({loaders: Loader[];loadOptions: any;}): Promise<AsyncGenerator> {- loaders = [JSONLoader, CSVLoader, GeoArrowLoader, ParquetArrowLoader, ...loaders];+ loaders = await getKeplerLoaders(file, loaders);const hasExtension = /\.[a-z0-9]+$/i.test(file.name);const mimeType = !hasExtension && file.type ? file.type : undefined;loadOptions = {diff --git a/src/processors/src/index.ts b/src/processors/src/index.tsindex dc99eaccf..b91a6808c 100644--- a/src/processors/src/index.ts+++ b/src/processors/src/index.ts@@ -5,3 +5,5 @@ export * from './data-processor';export * from './file-handler';export * from './remote-file';export * from './types';+export * from './kepler-csv-loader';+export * from './loader-registry';diff --git a/src/processors/src/kepler-csv-loader.ts b/src/processors/src/kepler-csv-loader.tsnew file mode 100644index 000000000..6086cc6dd--- /dev/null+++ b/src/processors/src/kepler-csv-loader.ts@@ -0,0 +1,89 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import type {LoaderContext, LoaderWithParser} from '@loaders.gl/loader-utils';+import type {CSVLoaderOptions} from '@loaders.gl/csv';++const KEPLER_CSV_OPTIONS: CSVLoaderOptions = {+ csv: {+ dynamicTyping: false,+ delimitersToGuess: [',', '\t', ';', '|']+ }+};++type KeplerCSVLoaderType = LoaderWithParser<any, any, CSVLoaderOptions>;++let csvLoaderPromise: Promise<KeplerCSVLoaderType> | null = null;++async function getCSVLoader(): Promise<KeplerCSVLoaderType> {+ csvLoaderPromise ||= import('@loaders.gl/csv').then(+ module => module.CSVLoader as KeplerCSVLoaderType+ );+ return csvLoaderPromise;+}++function getOptions(options?: CSVLoaderOptions): CSVLoaderOptions {+ return {+ ...options,+ csv: {+ ...KEPLER_CSV_OPTIONS.csv,+ ...options?.csv,+ // Kepler performs its own type inference after parsing.+ dynamicTyping: false,+ delimitersToGuess: [',', '\t', ';', '|']+ }+ };+}++/**+ * CSV loader used by Kepler's asynchronous file-loading path.+ *+ * The parser implementation is dynamically imported on first use. Kepler's+ * existing row processor remains responsible for null normalization and type+ * inference, preserving the public processCsvData contract.+ */+export const KeplerCSVLoader: KeplerCSVLoaderType = {+ name: 'Kepler CSV',+ id: 'csv',+ module: 'csv',+ version: '1.0.0',+ extensions: ['csv', 'tsv', 'dsv'],+ mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],+ category: 'table',+ text: true,+ options: KEPLER_CSV_OPTIONS,++ parse: async (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions, context?: LoaderContext) => {+ const loader = await getCSVLoader();+ if (!loader.parse) {+ throw new Error('KeplerCSVLoader: CSV loader does not support parse');+ }+ return loader.parse(arrayBuffer, getOptions(options), context);+ },++ parseInBatches: (+ iterator:+ | AsyncIterable<ArrayBufferLike | ArrayBufferView>+ | Iterable<ArrayBufferLike | ArrayBufferView>,+ options?: CSVLoaderOptions,+ context?: LoaderContext+ ) => {+ const batches = (async function* () {+ const loader = await getCSVLoader();+ const output = loader.parseInBatches?.(iterator, getOptions(options), context);+ if (!output) {+ throw new Error('KeplerCSVLoader: CSV loader does not support parseInBatches');+ }+ yield* output;+ })();+ return batches;+ },++ parseText: async (text: string, options?: CSVLoaderOptions, context?: LoaderContext) => {+ const loader = await getCSVLoader();+ if (!loader.parseText) {+ throw new Error('KeplerCSVLoader: CSV loader does not support parseText');+ }+ return loader.parseText(text, getOptions(options), context);+ }+};diff --git a/src/processors/src/loader-registry.ts b/src/processors/src/loader-registry.tsnew file mode 100644index 000000000..189d00b31--- /dev/null+++ b/src/processors/src/loader-registry.ts@@ -0,0 +1,78 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import type {Loader} from '@loaders.gl/loader-utils';++import {KeplerCSVLoader} from './kepler-csv-loader';++export type KeplerLoaderEntry = {+ id: string;+ extensions: string[];+ mimeTypes: string[];+ load: () => Promise<Loader>;+};++type FileMetadata = {+ name?: string;+ type?: string;+};++function getFileExtension(fileName = ''): string {+ const match = fileName.match(/\.([a-z0-9]+)$/i);+ return match ? match[1].toLowerCase() : '';+}++const DEFAULT_LOADER_ENTRIES: KeplerLoaderEntry[] = [+ {+ id: 'csv',+ extensions: ['csv', 'tsv', 'dsv'],+ mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],+ load: async () => KeplerCSVLoader+ },+ {+ id: 'json',+ extensions: ['json', 'geojson'],+ mimeTypes: ['application/json', 'application/geo+json'],+ load: async () => (await import('@loaders.gl/json')).JSONLoader+ },+ {+ id: 'arrow',+ extensions: ['arrow', 'feather'],+ mimeTypes: ['application/vnd.apache.arrow.file', 'application/vnd.apache.arrow.stream'],+ load: async () => (await import('@loaders.gl/arrow')).GeoArrowLoader+ },+ {+ id: 'parquet',+ extensions: ['parquet'],+ mimeTypes: ['application/vnd.apache.parquet'],+ load: async () => (await import('@loaders.gl/parquet')).ParquetArrowLoader+ }+];++export const KEPLER_LOADER_ENTRIES = DEFAULT_LOADER_ENTRIES;++function matchesFile(entry: KeplerLoaderEntry, file: FileMetadata): boolean {+ const extension = getFileExtension(file.name);+ const mimeType = file.type?.split(';')[0].trim().toLowerCase();+ return Boolean(+ (extension && entry.extensions.includes(extension)) ||+ (mimeType && entry.mimeTypes.includes(mimeType))+ );+}++/**+ * Resolve the loaders needed for a file. Loader modules are imported only for+ * matching extensions/MIME types; extensionless or unknown files retain core's+ * content-based selection by loading all default candidates.+ */+export async function getKeplerLoaders(+ file: FileMetadata,+ customLoaders: Loader[] = []+): Promise<Loader[]> {+ const matchingEntries = KEPLER_LOADER_ENTRIES.filter(entry => matchesFile(entry, file));+ const entries = matchingEntries.length ? matchingEntries : KEPLER_LOADER_ENTRIES;+ const defaultLoaders = await Promise.all(entries.map(entry => entry.load()));+ const customLoaderIds = new Set(customLoaders.map(loader => loader.id));++ return [...customLoaders, ...defaultLoaders.filter(loader => !customLoaderIds.has(loader.id))];+}diff --git a/src/table/src/tileset/tileset-utils.ts b/src/table/src/tileset/tileset-utils.tsindex 83b42e29d..3ef0e28e9 100644--- a/src/table/src/tileset/tileset-utils.ts+++ b/src/table/src/tileset/tileset-utils.ts@@ -1,8 +1,12 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project+import {parse} from '@loaders.gl/core';import {TileJSONLoader, TileJSON} from '@loaders.gl/mvt';+// TileJSONLoader omits the spec `attribution` field. parseVectorMetadata reads `attributions`.+type TileJSONWithAttributions = TileJSON & {attributions?: string[]};+/*** MVTSource in current loaders ignores attribution* TODO: remove once MVTSource is ready@@ -26,7 +30,11 @@ export async function getMVTMetadata(metadataURL: string | null): Promise<TileJSreturn null;}const tileJSON = await response.text();- const metadata = TileJSONLoader.parseTextSync?.(tileJSON) || null;+ // parse() does not infer TileJSONLoader's data type.+ const metadata = (await parse(tileJSON, TileJSONLoader)) as TileJSONWithAttributions | null;+ if (!metadata) {+ return null;+ }const rawMetadata = JSON.parse(tileJSON);if (rawMetadata?.attribution) {diff --git a/test/node/processors/index.js b/test/node/processors/index.jsindex 1d0970ead..71823c083 100644--- a/test/node/processors/index.js+++ b/test/node/processors/index.js@@ -3,3 +3,4 @@import './file-handler-test';import './remote-file-test';+import './loader-registry-test';diff --git a/test/node/processors/loader-registry-test.js b/test/node/processors/loader-registry-test.jsnew file mode 100644index 000000000..d5480104d--- /dev/null+++ b/test/node/processors/loader-registry-test.js@@ -0,0 +1,54 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import test from 'tape';+import {getKeplerLoaders, KeplerCSVLoader, readFileInBatches} from '@kepler.gl/processors';++test('#loader-registry -> resolves matching loaders lazily', async t => {+ const loaders = await getKeplerLoaders({name: 'data.csv', type: ''});++ t.equal(loaders.length, 1, 'only the matching default loader should be resolved');+ t.equal(loaders[0], KeplerCSVLoader, 'CSV should resolve to the Kepler CSV adapter');+ t.equal(typeof loaders[0].parseInBatches, 'function', 'adapter should expose parseInBatches');+ t.end();+});++test('#loader-registry -> resolves JSON loaders from MIME metadata', async t => {+ const loaders = await getKeplerLoaders({name: 'data', type: 'application/json; charset=utf-8'});++ t.equal(loaders.length, 1, 'only the matching default loader should be resolved');+ t.equal(loaders[0].id, 'json', 'JSON should be selected from the MIME type');+ t.end();+});++test('#loader-registry -> custom loaders take precedence', async t => {+ const customLoader = {...KeplerCSVLoader, name: 'Custom CSV'};+ const loaders = await getKeplerLoaders({name: 'data.csv', type: ''}, [customLoader]);++ t.equal(loaders.length, 1, 'the default loader should be deduplicated');+ t.equal(loaders[0], customLoader, 'custom loader should be first and authoritative');+ t.end();+});++test('#loader-registry -> CSV parser is async and dynamically loaded', async t => {+ const table = await KeplerCSVLoader.parseText('name,value\nalpha,1\n');++ t.equal(table.shape, 'object-row-table', 'CSV should return a loaders.gl table');+ t.deepEqual(table.data, [{name: 'alpha', value: '1'}], 'CSV values should remain strings');+ t.end();+});++test('#loader-registry -> file loading uses the async loader path', async t => {+ const file = new File(['name,value\nalpha,1\n'], 'data.csv', {type: 'text/csv'});+ const generator = await readFileInBatches({file, fileCache: [], loaders: [], loadOptions: {}});+ const batches = [];++ for await (const batch of generator) {+ batches.push(batch);+ }++ const dataBatch = batches.find(batch => batch.batchType === 'data');+ t.ok(dataBatch, 'should yield a data batch');+ t.deepEqual(dataBatch.data, [{name: 'alpha', value: '1'}], 'should preserve CSV rows');+ t.end();+});