Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

feat: add async loaders.gl 4.4 loader registry (#3682)

Ib Green4fce8abdbb4f
diff --git a/src/layers/src/layer-utils.ts b/src/layers/src/layer-utils.ts
index ee074093e..30390abde 100644
--- a/src/layers/src/layer-utils.ts
+++ b/src/layers/src/layer-utils.ts
@@ -21,6 +21,7 @@ import {getBinaryGeometriesFromArrow, BinaryGeometriesFromArrowOptions} from '@l
import {updateBoundsFromGeoArrowSamples} from '@loaders.gl/geoarrow';
import {convertGeoArrowGeometryToGeoJSON} from '@loaders.gl/gis';
+import {parseSync} from '@loaders.gl/core';
import {WKBLoader} from '@loaders.gl/wkt';
import {geojsonToBinary} from '@loaders.gl/gis';
import {
@@ -112,7 +113,7 @@ function getBinaryGeometriesFromWKBArrow(
if (chunk.valueOffsets[i + 1] - chunk.valueOffsets[i] > 0) {
const valuesSlice = chunk.values.slice(chunk.valueOffsets[i], chunk.valueOffsets[i + 1]);
- const geometry = WKBLoader?.parseSync?.(valuesSlice.buffer, {
+ const geometry = parseSync(valuesSlice.buffer, WKBLoader, {
wkb: {shape: 'geojson-geometry'}
}) as Geometry;
const feature: Feature = {
diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.ts
index 9c48a148c..0fd1756aa 100644
--- a/src/processors/src/file-handler.ts
+++ b/src/processors/src/file-handler.ts
@@ -2,10 +2,6 @@
// Copyright contributors to the kepler.gl project
import {parseInBatches} from '@loaders.gl/core';
-import {JSONLoader, _JSONPath} from '@loaders.gl/json';
-import {CSVLoader} from '@loaders.gl/csv';
-import {GeoArrowLoader} from '@loaders.gl/arrow';
-import {ParquetArrowLoader} from '@loaders.gl/parquet';
import {Loader} from '@loaders.gl/loader-utils';
import {
isPlainObject,
@@ -28,6 +24,7 @@ import {
} from './data-processor';
import {FileCacheItem, ValidKeplerGlMap} from './types';
+import {getKeplerLoaders} from './loader-registry';
const BATCH_TYPE = {
METADATA: 'metadata',
@@ -184,6 +181,7 @@ export async function* readBatch(
// Set the streamed data correctly is Batch json path is set
// and the path streamed is not the top level object (jsonpath = '$')
if (batch.jsonpath && batch.jsonpath.length > 1) {
+ const {_JSONPath} = await import('@loaders.gl/json');
const streamingPath = new _JSONPath(batch.jsonpath);
streamingPath.setFieldAtPath(result, batches);
} else if (batch.jsonpath && batch.jsonpath.length === 1) {
@@ -231,7 +229,7 @@ export async function readFileInBatches({
loaders: Loader[];
loadOptions: any;
}): Promise<AsyncGenerator> {
- loaders = [JSONLoader, CSVLoader, GeoArrowLoader, ParquetArrowLoader, ...loaders];
+ loaders = await getKeplerLoaders(file, loaders);
const hasExtension = /\.[a-z0-9]+$/i.test(file.name);
const mimeType = !hasExtension && file.type ? file.type : undefined;
loadOptions = {
diff --git a/src/processors/src/index.ts b/src/processors/src/index.ts
index dc99eaccf..b91a6808c 100644
--- a/src/processors/src/index.ts
+++ b/src/processors/src/index.ts
@@ -5,3 +5,5 @@ export * from './data-processor';
export * from './file-handler';
export * from './remote-file';
export * from './types';
+export * from './kepler-csv-loader';
+export * from './loader-registry';
diff --git a/src/processors/src/kepler-csv-loader.ts b/src/processors/src/kepler-csv-loader.ts
new file mode 100644
index 000000000..6086cc6dd
--- /dev/null
+++ b/src/processors/src/kepler-csv-loader.ts
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import type {LoaderContext, LoaderWithParser} from '@loaders.gl/loader-utils';
+import type {CSVLoaderOptions} from '@loaders.gl/csv';
+
+const KEPLER_CSV_OPTIONS: CSVLoaderOptions = {
+ csv: {
+ dynamicTyping: false,
+ delimitersToGuess: [',', '\t', ';', '|']
+ }
+};
+
+type KeplerCSVLoaderType = LoaderWithParser<any, any, CSVLoaderOptions>;
+
+let csvLoaderPromise: Promise<KeplerCSVLoaderType> | null = null;
+
+async function getCSVLoader(): Promise<KeplerCSVLoaderType> {
+ csvLoaderPromise ||= import('@loaders.gl/csv').then(
+ module => module.CSVLoader as KeplerCSVLoaderType
+ );
+ return csvLoaderPromise;
+}
+
+function getOptions(options?: CSVLoaderOptions): CSVLoaderOptions {
+ return {
+ ...options,
+ csv: {
+ ...KEPLER_CSV_OPTIONS.csv,
+ ...options?.csv,
+ // Kepler performs its own type inference after parsing.
+ dynamicTyping: false,
+ delimitersToGuess: [',', '\t', ';', '|']
+ }
+ };
+}
+
+/**
+ * CSV loader used by Kepler's asynchronous file-loading path.
+ *
+ * The parser implementation is dynamically imported on first use. Kepler's
+ * existing row processor remains responsible for null normalization and type
+ * inference, preserving the public processCsvData contract.
+ */
+export const KeplerCSVLoader: KeplerCSVLoaderType = {
+ name: 'Kepler CSV',
+ id: 'csv',
+ module: 'csv',
+ version: '1.0.0',
+ extensions: ['csv', 'tsv', 'dsv'],
+ mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
+ category: 'table',
+ text: true,
+ options: KEPLER_CSV_OPTIONS,
+
+ parse: async (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions, context?: LoaderContext) => {
+ const loader = await getCSVLoader();
+ if (!loader.parse) {
+ throw new Error('KeplerCSVLoader: CSV loader does not support parse');
+ }
+ return loader.parse(arrayBuffer, getOptions(options), context);
+ },
+
+ parseInBatches: (
+ iterator:
+ | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+ | Iterable<ArrayBufferLike | ArrayBufferView>,
+ options?: CSVLoaderOptions,
+ context?: LoaderContext
+ ) => {
+ const batches = (async function* () {
+ const loader = await getCSVLoader();
+ const output = loader.parseInBatches?.(iterator, getOptions(options), context);
+ if (!output) {
+ throw new Error('KeplerCSVLoader: CSV loader does not support parseInBatches');
+ }
+ yield* output;
+ })();
+ return batches;
+ },
+
+ parseText: async (text: string, options?: CSVLoaderOptions, context?: LoaderContext) => {
+ const loader = await getCSVLoader();
+ if (!loader.parseText) {
+ throw new Error('KeplerCSVLoader: CSV loader does not support parseText');
+ }
+ return loader.parseText(text, getOptions(options), context);
+ }
+};
diff --git a/src/processors/src/loader-registry.ts b/src/processors/src/loader-registry.ts
new file mode 100644
index 000000000..189d00b31
--- /dev/null
+++ b/src/processors/src/loader-registry.ts
@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import type {Loader} from '@loaders.gl/loader-utils';
+
+import {KeplerCSVLoader} from './kepler-csv-loader';
+
+export type KeplerLoaderEntry = {
+ id: string;
+ extensions: string[];
+ mimeTypes: string[];
+ load: () => Promise<Loader>;
+};
+
+type FileMetadata = {
+ name?: string;
+ type?: string;
+};
+
+function getFileExtension(fileName = ''): string {
+ const match = fileName.match(/\.([a-z0-9]+)$/i);
+ return match ? match[1].toLowerCase() : '';
+}
+
+const DEFAULT_LOADER_ENTRIES: KeplerLoaderEntry[] = [
+ {
+ id: 'csv',
+ extensions: ['csv', 'tsv', 'dsv'],
+ mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
+ load: async () => KeplerCSVLoader
+ },
+ {
+ id: 'json',
+ extensions: ['json', 'geojson'],
+ mimeTypes: ['application/json', 'application/geo+json'],
+ load: async () => (await import('@loaders.gl/json')).JSONLoader
+ },
+ {
+ id: 'arrow',
+ extensions: ['arrow', 'feather'],
+ mimeTypes: ['application/vnd.apache.arrow.file', 'application/vnd.apache.arrow.stream'],
+ load: async () => (await import('@loaders.gl/arrow')).GeoArrowLoader
+ },
+ {
+ id: 'parquet',
+ extensions: ['parquet'],
+ mimeTypes: ['application/vnd.apache.parquet'],
+ load: async () => (await import('@loaders.gl/parquet')).ParquetArrowLoader
+ }
+];
+
+export const KEPLER_LOADER_ENTRIES = DEFAULT_LOADER_ENTRIES;
+
+function matchesFile(entry: KeplerLoaderEntry, file: FileMetadata): boolean {
+ const extension = getFileExtension(file.name);
+ const mimeType = file.type?.split(';')[0].trim().toLowerCase();
+ return Boolean(
+ (extension && entry.extensions.includes(extension)) ||
+ (mimeType && entry.mimeTypes.includes(mimeType))
+ );
+}
+
+/**
+ * Resolve the loaders needed for a file. Loader modules are imported only for
+ * matching extensions/MIME types; extensionless or unknown files retain core's
+ * content-based selection by loading all default candidates.
+ */
+export async function getKeplerLoaders(
+ file: FileMetadata,
+ customLoaders: Loader[] = []
+): Promise<Loader[]> {
+ const matchingEntries = KEPLER_LOADER_ENTRIES.filter(entry => matchesFile(entry, file));
+ const entries = matchingEntries.length ? matchingEntries : KEPLER_LOADER_ENTRIES;
+ const defaultLoaders = await Promise.all(entries.map(entry => entry.load()));
+ const customLoaderIds = new Set(customLoaders.map(loader => loader.id));
+
+ return [...customLoaders, ...defaultLoaders.filter(loader => !customLoaderIds.has(loader.id))];
+}
diff --git a/src/table/src/tileset/tileset-utils.ts b/src/table/src/tileset/tileset-utils.ts
index 83b42e29d..3ef0e28e9 100644
--- a/src/table/src/tileset/tileset-utils.ts
+++ b/src/table/src/tileset/tileset-utils.ts
@@ -1,8 +1,12 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
+import {parse} from '@loaders.gl/core';
import {TileJSONLoader, TileJSON} from '@loaders.gl/mvt';
+// TileJSONLoader omits the spec `attribution` field. parseVectorMetadata reads `attributions`.
+type TileJSONWithAttributions = TileJSON & {attributions?: string[]};
+
/**
* MVTSource in current loaders ignores attribution
* TODO: remove once MVTSource is ready
@@ -26,7 +30,11 @@ export async function getMVTMetadata(metadataURL: string | null): Promise<TileJS
return null;
}
const tileJSON = await response.text();
- const metadata = TileJSONLoader.parseTextSync?.(tileJSON) || null;
+ // parse() does not infer TileJSONLoader's data type.
+ const metadata = (await parse(tileJSON, TileJSONLoader)) as TileJSONWithAttributions | null;
+ if (!metadata) {
+ return null;
+ }
const rawMetadata = JSON.parse(tileJSON);
if (rawMetadata?.attribution) {
diff --git a/test/node/processors/index.js b/test/node/processors/index.js
index 1d0970ead..71823c083 100644
--- a/test/node/processors/index.js
+++ b/test/node/processors/index.js
@@ -3,3 +3,4 @@
import './file-handler-test';
import './remote-file-test';
+import './loader-registry-test';
diff --git a/test/node/processors/loader-registry-test.js b/test/node/processors/loader-registry-test.js
new file mode 100644
index 000000000..d5480104d
--- /dev/null
+++ b/test/node/processors/loader-registry-test.js
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import test from 'tape';
+import {getKeplerLoaders, KeplerCSVLoader, readFileInBatches} from '@kepler.gl/processors';
+
+test('#loader-registry -> resolves matching loaders lazily', async t => {
+ const loaders = await getKeplerLoaders({name: 'data.csv', type: ''});
+
+ t.equal(loaders.length, 1, 'only the matching default loader should be resolved');
+ t.equal(loaders[0], KeplerCSVLoader, 'CSV should resolve to the Kepler CSV adapter');
+ t.equal(typeof loaders[0].parseInBatches, 'function', 'adapter should expose parseInBatches');
+ t.end();
+});
+
+test('#loader-registry -> resolves JSON loaders from MIME metadata', async t => {
+ const loaders = await getKeplerLoaders({name: 'data', type: 'application/json; charset=utf-8'});
+
+ t.equal(loaders.length, 1, 'only the matching default loader should be resolved');
+ t.equal(loaders[0].id, 'json', 'JSON should be selected from the MIME type');
+ t.end();
+});
+
+test('#loader-registry -> custom loaders take precedence', async t => {
+ const customLoader = {...KeplerCSVLoader, name: 'Custom CSV'};
+ const loaders = await getKeplerLoaders({name: 'data.csv', type: ''}, [customLoader]);
+
+ t.equal(loaders.length, 1, 'the default loader should be deduplicated');
+ t.equal(loaders[0], customLoader, 'custom loader should be first and authoritative');
+ t.end();
+});
+
+test('#loader-registry -> CSV parser is async and dynamically loaded', async t => {
+ const table = await KeplerCSVLoader.parseText('name,value\nalpha,1\n');
+
+ t.equal(table.shape, 'object-row-table', 'CSV should return a loaders.gl table');
+ t.deepEqual(table.data, [{name: 'alpha', value: '1'}], 'CSV values should remain strings');
+ t.end();
+});
+
+test('#loader-registry -> file loading uses the async loader path', async t => {
+ const file = new File(['name,value\nalpha,1\n'], 'data.csv', {type: 'text/csv'});
+ const generator = await readFileInBatches({file, fileCache: [], loaders: [], loadOptions: {}});
+ const batches = [];
+
+ for await (const batch of generator) {
+ batches.push(batch);
+ }
+
+ const dataBatch = batches.find(batch => batch.batchType === 'data');
+ t.ok(dataBatch, 'should yield a data batch');
+ t.deepEqual(dataBatch.data, [{name: 'alpha', value: '1'}], 'should preserve CSV rows');
+ t.end();
+});