Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
feat(dataset): load mixed zip files as separate datasets (#3792)
Igor Dykhta5c8f90d4318d
diff --git a/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md b/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.mdindex 813d4635d..a55a0d289 100644--- a/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md+++ b/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md@@ -11,7 +11,7 @@| | ||---|---|-| **Local files** | <span style="font-weight:normal">Upload CSV, GeoJSON, GeoJSONL, KML, GPX, TCX, Shapefile, Excel, FlatGeobuf, Arrow, or Parquet files. Files are parsed in your browser, so available memory limits how large a file you can load. For data that does not fit in memory, use a tileset instead.<span> |+| **Local files** | <span style="font-weight:normal">Upload CSV, GeoJSON, GeoJSONL, KML, GPX, TCX, Shapefile, Excel, FlatGeobuf, Arrow, or Parquet files, or a zip of those files. Files are parsed in your browser, so available memory limits how large a file you can load. For data that does not fit in memory, use a tileset instead.<span> || **From URL** | Directly load data or map json by pasting a remote URL. Supported extensions include `csv`, `geojson`, `json`, `geojsonl`, `kml`, `gpx`, `tcx`, `shp`, `xlsx`, `fgb`, `arrow`, and `parquet`. Make sure the url contains the file extension. CORS policy must be defined on your custom url domain. || **Sample data** | Load one of kepler.gl’s sample datasets. The sample map data and config are directly loaded from [kepler.gl-data github][kepler.gl-data-github] repo |@@ -29,6 +29,7 @@ Geometry coordinates should be presented with a geographic coordinate reference- [GPX](#gpx)- [TCX](#tcx)- [Shapefile](#shapefile)+ - [Zip](#zip)- [Excel](#excel)- [FlatGeobuf](#flatgeobuf)- [GeoArrow](#geoarrow)@@ -225,7 +226,13 @@ kepler.gl accepts Garmin `.tcx` activity files. Track points are converted to Ge### Shapefile-kepler.gl accepts ESRI Shapefiles. Drop a `.shp` together with its sidecars (`.dbf`, `.shx`, and optionally `.prj` / `.cpg`), or a `.zip` that contains those files at the top level. Features are converted to GeoJSON and visualized with a **Polygon** layer. If a `.prj` file is present, geometries are reprojected to WGS84. Nested folders inside the zip are ignored.+kepler.gl accepts ESRI Shapefiles. Drop a `.shp` together with its sidecars (`.dbf`, `.shx`, and optionally `.prj` / `.cpg`), or a `.zip` that contains one shapefile and those sidecars at the top level. A zip of a single shapefile loads as one dataset named after the zip. Features are converted to GeoJSON and visualized with a **Polygon** layer. If a `.prj` file is present, geometries are reprojected to WGS84. Nested folders inside the zip are ignored.++### Zip++A `.zip` can hold more than one dataset. Each supported file at the top level becomes its own dataset, named with the file name inside the archive. CSV, GeoJSON, GeoJSONL, KML, GPX, TCX, Excel, FlatGeobuf, Arrow, Parquet, and Shapefile are recognized. Shapefile sidecars stay with the `.shp` they belong to, and a zip of several shapefiles loads each one. Nested folders are ignored. Files kepler.gl cannot load are skipped. A zip with nothing supported reports an error.++A zip that contains a single shapefile and its sidecars stays one dataset named after the zip.### Exceldiff --git a/src/actions/src/action-types.ts b/src/actions/src/action-types.tsindex bcc79a3be..9ce1a9bde 100644--- a/src/actions/src/action-types.ts+++ b/src/actions/src/action-types.ts@@ -98,6 +98,7 @@ export const ActionTypes = {SET_FILTER_PLOT: `${ACTION_PREFIX}SET_FILTER_PLOT`,LOAD_FILES: `${ACTION_PREFIX}LOAD_FILES`,LOAD_NEXT_FILE: `${ACTION_PREFIX}LOAD_NEXT_FILE`,+ EXPAND_ZIP_ARCHIVE_SUCCESS: `${ACTION_PREFIX}EXPAND_ZIP_ARCHIVE_SUCCESS`,LOAD_BATCH_DATA_SUCCESS: `${ACTION_PREFIX}LOAD_BATCH_DATA_SUCCESS`,LOAD_FILE_STEP_SUCCESS: `${ACTION_PREFIX}LOAD_FILE_STEP_SUCCESS`,LOAD_FILES_ERR: `${ACTION_PREFIX}LOAD_FILES_ERR`,diff --git a/src/actions/src/vis-state-actions.ts b/src/actions/src/vis-state-actions.tsindex 5a0ccbd3c..317d39b40 100644--- a/src/actions/src/vis-state-actions.ts+++ b/src/actions/src/vis-state-actions.ts@@ -3,7 +3,7 @@// vis-state-reducerimport {default as ActionTypes} from './action-types';-import {FileCacheItem} from '@kepler.gl/processors';+import {FileCacheItem, ZipArchiveExpansion} from '@kepler.gl/processors';import {Layer, LayerBaseConfig} from '@kepler.gl/layers';import {GroupByOp, JoinOp, KeplerTable} from '@kepler.gl/table';import {@@ -1881,6 +1881,25 @@ export function loadNextFile(loadId?: number): {};}+export type ExpandZipArchiveSuccessAction = {+ loadId: number;+ archiveName: string;+ expansion: ZipArchiveExpansion;+};+/**+ * A zip in the load queue has been unpacked. The updater either parses a+ * single shapefile archive or enqueues one dataset per supported member.+ * @memberof visStateActions+ */+export function expandZipArchiveSuccess(+ payload: ExpandZipArchiveSuccessAction+): Merge<ExpandZipArchiveSuccessAction, {type: typeof ActionTypes.EXPAND_ZIP_ARCHIVE_SUCCESS}> {+ return {+ type: ActionTypes.EXPAND_ZIP_ARCHIVE_SUCCESS,+ ...payload+ };+}+export type loadFilesSuccessUpdaterAction = {result: FileCacheItem[];options?: AddDataToMapOptions;@@ -2008,6 +2027,8 @@ export function loadFileStepSuccess({export type LoadFilesErrUpdaterAction = {fileName: string;error: any;+ /** Set when the failure belongs to one load, so a newer load can ignore it. */+ loadId?: number;};/*** Trigger loading file error@@ -2019,12 +2040,14 @@ export type LoadFilesErrUpdaterAction = {export function loadFilesErr(fileName: string,- error: any+ error: any,+ loadId?: number): Merge<LoadFilesErrUpdaterAction, {type: typeof ActionTypes.LOAD_FILES_ERR}> {return {type: ActionTypes.LOAD_FILES_ERR,fileName,- error+ error,+ ...(loadId == null ? {} : {loadId})};}diff --git a/src/components/src/common/file-uploader/file-upload.tsx b/src/components/src/common/file-uploader/file-upload.tsxindex b7674136d..e495e299f 100644--- a/src/components/src/common/file-uploader/file-upload.tsx+++ b/src/components/src/common/file-uploader/file-upload.tsx@@ -252,9 +252,41 @@ function extensionOf(name: string): string {return part && part !== name ? part : '';}+/**+ * A zip is only a dataset when parsing keeps its name (a single shapefile).+ * Once it has expanded into other files, drop the archive row so those files+ * are the only lines under "To add to map".+ */+function archiveExpandedAway(+ fileName: string,+ progress: FileLoadingProgress,+ stagedToAdd: Array<{info?: {label?: string}; metadata?: {source?: string}}> | null | undefined,+ fileLoading: unknown,+ droppedNames: Set<string>+): boolean {+ if (extensionOf(fileName).toLowerCase() !== 'zip') {+ return false;+ }+ if (progress[fileName]?.error) {+ return false;+ }+ const keptAsDataset = (stagedToAdd || []).some(+ item => !item?.metadata?.source && item?.info?.label === fileName+ );+ if (keptAsDataset || (fileLoading && progress[fileName])) {+ return false;+ }+ const unpackedName = (name?: string) => Boolean(name && !droppedNames.has(name));+ const unpackedFile = (stagedToAdd || []).some(+ item => !item?.metadata?.source && unpackedName(item?.info?.label)+ );+ const unpackedProgress = Object.keys(progress).some(unpackedName);+ return unpackedFile || unpackedProgress;+}+function fileListStatus(intl,- file: File,+ file: {size?: number},progress: FileLoadingProgress[string] | undefined,loading: boolean): Pick<UploadFileListItem, 'status' | 'percent' | 'isError' | 'isSuccess'> {@@ -471,7 +503,10 @@ function FileUploadFactory() {);});files.forEach(file => {- if (seen.has(file.name)) {+ if (+ seen.has(file.name) ||+ archiveExpandedAway(file.name, progress, stagedToAdd, fileLoading, localNames)+ ) {return;}seen.add(file.name);@@ -494,6 +529,26 @@ function FileUploadFactory() {));});+ // Files unpacked from a zip are not in the drop list. Show their progress+ // until they are staged under their own names.+ Object.values(progress).forEach(item => {+ if (!item?.fileName || seen.has(item.fileName)) {+ return;+ }+ seen.add(item.fileName);+ const status = fileListStatus(intl, {}, item, Boolean(fileLoading));+ uploadItems.push(+ selectionFor(+ {+ name: item.fileName,+ ext: extensionOf(item.fileName),+ ...status+ },+ item.fileName,+ Boolean(status.isSuccess) && !status.isError+ )+ );+ });errorFiles.forEach(name => {if (seen.has(name)) {return;diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.tsindex 08853b069..16a44c7ae 100644--- a/src/processors/src/file-handler.ts+++ b/src/processors/src/file-handler.ts@@ -377,6 +377,8 @@ export async function readFileInBatches({fileName?: string;}): Promise<AsyncGenerator> {const displayFileName = fileName || file.name;+ // Local drops expand mixed zips in the load queue before this runs. A `.zip`+ // that still arrives here is a shapefile archive (remote URL, or a direct call).if (isZipFileName(file.name) && isKeplerFileFormatAccepted('shp')) {const unzipped = await unzipShapefileArchive(file);const shapefile = unzipped.find(entry => getDroppedFileExtension(entry.name) === 'shp');diff --git a/src/processors/src/loader-registry.ts b/src/processors/src/loader-registry.tsindex 56bc8a0cf..ec8e192e7 100644--- a/src/processors/src/loader-registry.ts+++ b/src/processors/src/loader-registry.ts@@ -129,6 +129,23 @@ const DEFAULT_LOADER_ENTRIES: KeplerLoaderEntry[] = [export const KEPLER_LOADER_ENTRIES = DEFAULT_LOADER_ENTRIES;+const LOADABLE_DATASET_EXTENSIONS = new Set(+ DEFAULT_LOADER_ENTRIES.flatMap(entry => entry.extensions)+);++/**+ * True when `extension` is a dataset kepler.gl can parse (not a shapefile+ * sidecar, and not `.zip`, which is an archive rather than a dataset).+ * `acceptedFileFormats` still applies.+ */+export function isLoadableDatasetExtension(extension: string): boolean {+ const ext = extension.trim().toLowerCase();+ if (!LOADABLE_DATASET_EXTENSIONS.has(ext)) {+ return false;+ }+ return isKeplerFileFormatAccepted(ext);+}+/** Maps user-facing format ids and extensions to a loader-registry id. */const TOKEN_TO_LOADER_ID: Record<string, string> = {csv: 'csv',@@ -199,7 +216,13 @@ export function isKeplerFileFormatAccepted(token: string): boolean {if (!allowed) {return true;}- const id = TOKEN_TO_LOADER_ID[token.trim().toLowerCase()];+ const normalized = token.trim().toLowerCase();+ // A zip is an archive of whatever formats are allowed, not a shapefile alias.+ // Keep it droppable when any dataset format is accepted.+ if (normalized === 'zip') {+ return allowed.size > 0;+ }+ const id = TOKEN_TO_LOADER_ID[normalized];return Boolean(id && allowed.has(id));}diff --git a/src/processors/src/shapefile-files.ts b/src/processors/src/shapefile-files.tsindex fd4c48bcf..2e6f2f9bd 100644--- a/src/processors/src/shapefile-files.ts+++ b/src/processors/src/shapefile-files.ts@@ -3,6 +3,8 @@import {_BrowserFileSystem, parse} from '@loaders.gl/core';+import {isLoadableDatasetExtension} from './loader-registry';+export const SHAPEFILE_SIDECAR_EXTENSIONS = ['dbf', 'shx', 'prj', 'cpg'] as const;export const SHAPEFILE_SOURCE_EXTENSIONS = ['shp', 'zip'] as const;@@ -62,29 +64,96 @@ function fileNameFromArchivePath(path: string): string {}/**- * Unpack a zip that contains a shapefile. Only top-level .shp plus sidecars- * are returned; nested folders are ignored.+ * Top-level members of a zip. Nested folders are ignored. Directory entries+ * are skipped.*/-export async function unzipShapefileArchive(zipFile: File): Promise<File[]> {+async function listTopLevelZipFiles(zipFile: File): Promise<File[]> {const {ZipLoader} = await import('@loaders.gl/zip');const zipData = await zipFile.arrayBuffer();const archive = (await parse(zipData, ZipLoader, {worker: false})) as Record<string, ArrayBuffer>;const files: File[] = [];for (const [path, buffer] of Object.entries(archive || {})) {- if (path.includes('/')) {+ if (!path || path.endsWith('/') || path.includes('/')) {continue;}const name = fileNameFromArchivePath(path);- const ext = getDroppedFileExtension(name);- if (ext === 'shp' || (SHAPEFILE_SIDECAR_EXTENSIONS as readonly string[]).includes(ext)) {- files.push(new File([buffer], name));+ if (!name) {+ continue;}+ files.push(new File([buffer], name));}+ return files;+}++/**+ * Unpack a zip that contains a shapefile. Only top-level .shp plus sidecars+ * are returned; nested folders are ignored.+ */+export async function unzipShapefileArchive(zipFile: File): Promise<File[]> {+ const files = (await listTopLevelZipFiles(zipFile)).filter(file => {+ const ext = getDroppedFileExtension(file.name);+ return ext === 'shp' || (SHAPEFILE_SIDECAR_EXTENSIONS as readonly string[]).includes(ext);+ });+if (!files.some(file => getDroppedFileExtension(file.name) === 'shp')) {throw new Error('Zip archive does not contain a shapefile (.shp)');}return files;}++export type ZipArchiveExpansion =+ | {+ /** One shapefile and no other datasets. The load keeps the zip's name. */+ kind: 'shapefile';+ shapefile: File;+ companions: File[];+ }+ | {+ /** One dataset per supported top-level file, named from that file. */+ kind: 'datasets';+ datasets: File[];+ companions: File[];+ };++/**+ * Split a zip into datasets. A single shapefile plus sidecars stays one+ * shapefile archive. Any other mix (CSV, GeoJSON, several shapefiles, …)+ * becomes one file per supported member. Sidecars are returned as companions+ * and are not datasets. Nested folders and unsupported files are skipped.+ */+export async function expandZipArchive(+ zipFile: File,+ extraExtensions: string[] = []+): Promise<ZipArchiveExpansion> {+ const extra = new Set(+ extraExtensions+ .map(extension => extension.replace(/^\./, '').trim().toLowerCase())+ .filter(extension => extension && extension !== 'zip')+ );+ const entries = await listTopLevelZipFiles(zipFile);+ const companions: File[] = [];+ const datasets: File[] = [];++ for (const file of entries) {+ if (isShapefileSidecarFileName(file.name)) {+ companions.push(file);+ continue;+ }+ const ext = getDroppedFileExtension(file.name);+ if (isLoadableDatasetExtension(ext) || extra.has(ext)) {+ datasets.push(file);+ }+ }++ const shapefiles = datasets.filter(file => getDroppedFileExtension(file.name) === 'shp');+ if (shapefiles.length === 1 && datasets.length === 1) {+ return {kind: 'shapefile', shapefile: shapefiles[0], companions};+ }+ if (!datasets.length) {+ throw new Error('Zip archive does not contain a supported dataset');+ }+ return {kind: 'datasets', datasets, companions};+}diff --git a/src/reducers/src/vis-state-selectors.ts b/src/reducers/src/vis-state-selectors.tsindex ae3d5b72f..ec9e6f108 100644--- a/src/reducers/src/vis-state-selectors.ts+++ b/src/reducers/src/vis-state-selectors.ts@@ -34,9 +34,10 @@ const DEFAULT_FILE_EXTENSIONS = ['xlsb','fgb'];-// One chip per format family. Aliases (tsv, jsonl, ndgeojson, zip, shapefile,+// One chip per format family. Aliases (tsv, jsonl, ndgeojson, shapefile,// excel, flatgeobuf, sidecars, …) stay accepted via DEFAULT_FILE_EXTENSIONS-// and listed in DISPLAY_FILE_FORMATS, but are not shown as extra icons.+// and are not shown as extra icons. Zip is its own format name: an archive+// can hold any of these datasets, not only a shapefile.const DISPLAY_FILE_EXTENSIONS = ['csv','json',@@ -59,6 +60,7 @@ const DISPLAY_FILE_FORMATS = ['GPX','TCX','Shapefile',+ 'Zip','Excel','FlatGeobuf'];diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.tsindex 41102d2f5..db4d526fe 100644--- a/src/reducers/src/vis-state-updaters.ts+++ b/src/reducers/src/vis-state-updaters.ts@@ -18,6 +18,7 @@ import type {TaskDescriptor} from '@kepler.gl/tasks';import {DELAY_TASK,ACTION_TASK,+ EXPAND_ZIP_TASK,LOAD_FILE_TASK,PROCESS_FILE_DATA,UNWRAP_TASK@@ -40,6 +41,7 @@ import {loadFilesErr,loadFilesSuccess,loadNextFile,+ expandZipArchiveSuccess,nextFileBatch,setFilter,processFileContent,@@ -157,7 +159,13 @@ import {} from './vis-state-merger';import KeplerGLSchema, {Merger, PostMergerPayload, VisState} from '@kepler.gl/schemas';-import {getFilesToParse, loadExternallyHostedDataset, processGeojson} from '@kepler.gl/processors';+import {+ getFilesToParse,+ isZipFileName,+ loadExternallyHostedDataset,+ processGeojson,+ ZipArchiveExpansion+} from '@kepler.gl/processors';import {Filter,@@ -4656,6 +4664,20 @@ export function loadNextFileUpdater(state: VisState, action?: {loadId?: number})});const {loaders, loadOptions} = state;+ if (isZipFileName(file.name)) {+ return withTask(+ stateWithProgress,+ EXPAND_ZIP_TASK({file, extensions: datasetExtensionsFromLoaders(loaders)}).bimap(+ expansion =>+ expandZipArchiveSuccess({+ loadId: currentLoadId,+ archiveName: file.name,+ expansion+ }),+ err => loadFilesErr(file.name, err, currentLoadId)+ )+ );+ }return withTask(stateWithProgress,makeLoadFileTask(@@ -4669,21 +4691,118 @@ export function loadNextFileUpdater(state: VisState, action?: {loadId?: number}));}+function datasetExtensionsFromLoaders(+ loaders: Array<{extensions?: readonly string[]}> = []+): string[] {+ const extensions: string[] = [];+ const seen = new Set<string>();+ for (const loader of loaders) {+ for (const extension of loader?.extensions || []) {+ const ext = String(extension).replace(/^\./, '').trim().toLowerCase();+ if (!ext || ext === 'zip' || seen.has(ext)) {+ continue;+ }+ seen.add(ext);+ extensions.push(ext);+ }+ }+ return extensions;+}++function companionFilesForArchive(expansion: ZipArchiveExpansion, existing?: File[]): File[] {+ const extracted =+ expansion.kind === 'shapefile'+ ? [expansion.shapefile, ...expansion.companions]+ : [...expansion.datasets, ...expansion.companions];+ return [...extracted, ...(existing || [])];+}++function withoutFileLoadingProgressEntry(+ progress: VisState['fileLoadingProgress'],+ fileName: string+): VisState['fileLoadingProgress'] {+ if (!progress || !Object.prototype.hasOwnProperty.call(progress, fileName)) {+ return progress;+ }+ const next = {...progress};+ delete next[fileName];+ return next;+}++/**+ * Apply an unpacked zip. One shapefile keeps the archive name. Any other+ * mix is queued as one dataset per supported member.+ */+export function expandZipArchiveSuccessUpdater(+ state: VisState,+ action: VisStateActions.ExpandZipArchiveSuccessAction+): VisState {+ const loading = state.fileLoading;+ if (!loading || loading.loadId !== action.loadId) {+ return state;+ }+ const {archiveName, expansion} = action;+ const companionFiles = companionFilesForArchive(expansion, loading.companionFiles);++ if (expansion.kind === 'shapefile') {+ const nextState = pick_('fileLoading')(merge_({companionFiles}))(state);+ const {loaders, loadOptions} = state;+ return withTask(+ nextState,+ makeLoadFileTask(+ expansion.shapefile,+ nextState.fileLoading && nextState.fileLoading.fileCache,+ loaders,+ loadOptions,+ companionFiles,+ loading.options,+ archiveName+ )+ );+ }++ const datasets = expansion.datasets;+ const filesToLoad = [...datasets, ...Array.from(loading.filesToLoad)];+ const fileLoadingProgress = datasets.reduce(+ (accu, file, index) => merge_(initialFileLoadingProgress(file, index))(accu),+ withoutFileLoadingProgressEntry(state.fileLoadingProgress, archiveName)+ );+ const nextState = {+ ...state,+ fileLoadingProgress,+ fileLoading: {+ ...loading,+ filesToLoad,+ companionFiles+ }+ };+ return loadNextFileUpdater(nextState, {loadId: action.loadId});+}+export function makeLoadFileTask(file,fileCache,loaders: Loader[] = [],loadOptions = {},companionFiles?: File[],- addDataOptions?: VisStateActions.LoadFilesOptions+ addDataOptions?: VisStateActions.LoadFilesOptions,+ fileName?: string) {- return LOAD_FILE_TASK({file, fileCache, loaders, loadOptions, companionFiles}).bimap(+ const displayName = fileName || file.name;+ return LOAD_FILE_TASK({+ file,+ fileCache,+ loaders,+ loadOptions,+ companionFiles,+ ...(fileName ? {fileName} : {})+ }).bimap(// prettier ignore// successgen =>nextFileBatch({gen,- fileName: file.name,+ fileName: displayName,onFinish: result =>processFileContent({content: result,@@ -4693,7 +4812,7 @@ export function makeLoadFileTask(}),// error- err => loadFilesErr(file.name, err)+ err => loadFilesErr(displayName, err));}@@ -4807,13 +4926,13 @@ export const clearStagedLoadedFilesUpdater = (state: VisState): VisState => ({export const loadFilesErrUpdater = (state: VisState,- {error, fileName}: VisStateActions.LoadFilesErrUpdaterAction+ {error, fileName, loadId: errorLoadId}: VisStateActions.LoadFilesErrUpdaterAction): VisState => {- // update ui with error message- Console.warn(error);- if (!state.fileLoading) {+ if (!state.fileLoading || (errorLoadId != null && state.fileLoading.loadId !== errorLoadId)) {return state;}+ // update ui with error message+ Console.warn(error);const loadId = state.fileLoading.loadId;const nextState = updateFileLoadingProgressUpdater(state, {diff --git a/src/reducers/src/vis-state.ts b/src/reducers/src/vis-state.tsindex 863900b8b..e1b3eac93 100644--- a/src/reducers/src/vis-state.ts+++ b/src/reducers/src/vis-state.ts@@ -58,6 +58,8 @@ const actionHandler = {[ActionTypes.LOAD_NEXT_FILE]: visStateUpdaters.loadNextFileUpdater,+ [ActionTypes.EXPAND_ZIP_ARCHIVE_SUCCESS]: visStateUpdaters.expandZipArchiveSuccessUpdater,+[ActionTypes.LOAD_FILE_STEP_SUCCESS]: visStateUpdaters.loadFileStepSuccessUpdater,[ActionTypes.MAP_CLICK]: visStateUpdaters.mapClickUpdater,diff --git a/src/tasks/src/index.ts b/src/tasks/src/index.tsindex 7774eda9c..a0f554ba4 100644--- a/src/tasks/src/index.ts+++ b/src/tasks/src/index.ts@@ -2,7 +2,7 @@// Copyright contributors to the kepler.gl projectimport Task, {taskCreator} from '@kepler.gl/tasks-core';-import {readFileInBatches, processFileData} from '@kepler.gl/processors';+import {readFileInBatches, processFileData, expandZipArchive} from '@kepler.gl/processors';export {Task,@@ -25,11 +25,24 @@ export {export type {TaskDescriptor} from '@kepler.gl/tasks-core';export const LOAD_FILE_TASK = Task.fromPromise(- ({file, fileCache, loaders, loadOptions, companionFiles}) =>- readFileInBatches({file, fileCache, loaders, loadOptions, companionFiles}),+ ({file, fileCache, loaders, loadOptions, companionFiles, fileName}) =>+ readFileInBatches({+ file,+ fileCache,+ loaders,+ loadOptions,+ companionFiles,+ ...(fileName ? {fileName} : {})+ }),'LOAD_FILE_TASK');+/** Unpack a zip before the load queue decides how many datasets it contains. */+export const EXPAND_ZIP_TASK = Task.fromPromise(+ ({file, extensions}: {file: File; extensions?: string[]}) => expandZipArchive(file, extensions),+ 'EXPAND_ZIP_TASK'+);+export const PROCESS_FILE_DATA = Task.fromPromise(processFileData, 'PROCESS_FILE_CONTENT');export const LOAD_MAP_STYLE_TASK = taskCreator(diff --git a/test/browser/components/common/file-uploader-test.js b/test/browser/components/common/file-uploader-test.jsindex 81d49cd47..3ba09dfb9 100644--- a/test/browser/components/common/file-uploader-test.js+++ b/test/browser/components/common/file-uploader-test.js@@ -175,6 +175,94 @@ test('Components -> FileUpload.onDrop keeps earlier files', t => {t.end();});+test('Components -> FileUpload zip expands to the files inside it', t => {+ const zip = [{type: 'application/zip', name: 'bundle.zip', size: 40}];+ const onFileUpload = sinon.spy();+ const wrapper = mountWithTheme(+ <IntlWrapper>+ <FileUpload onFileUpload={onFileUpload} fileExtensions={['csv', 'zip', 'json']} />+ </IntlWrapper>+ );+ wrapper+ .find('.file-uploader__file-drop')+ .at(0)+ .simulate('drop', {+ stopPropagation: () => {},+ dataTransfer: {types: ['Files'], files: zip}+ });++ wrapper.setProps({+ children: (+ <IntlWrapper>+ <FileUpload+ onFileUpload={onFileUpload}+ fileExtensions={['csv', 'zip', 'json']}+ fileLoading={false}+ fileLoadingProgress={{}}+ stagedToAdd={[{info: {label: 'places.csv', format: 'row'}}]}+ />+ </IntlWrapper>+ )+ });++ const keptFiles = wrapper.find(FileUpload).children().first().state().files;+ t.deepEqual(+ keptFiles.map(file => file.name),+ ['bundle.zip'],+ 'the dropped zip stays in the uploader state'+ );+ const listText = wrapper.find('.upload-file-list').text();+ t.ok(listText.includes('places.csv'), 'should list the csv that was inside the zip');+ t.notOk(listText.includes('bundle.zip'), 'should not list the zip once it has been expanded');+ t.equal(+ wrapper.find('.upload-file-list').find('Checkbox').length,+ 1,+ 'only the unpacked dataset can be added'+ );++ t.end();+});++test('Components -> FileUpload shapefile zip stays one row', t => {+ const zip = [{type: 'application/zip', name: 'places.zip', size: 40}];+ const wrapper = mountWithTheme(+ <IntlWrapper>+ <FileUpload onFileUpload={() => {}} fileExtensions={['zip', 'shp']} />+ </IntlWrapper>+ );+ wrapper+ .find('.file-uploader__file-drop')+ .at(0)+ .simulate('drop', {+ stopPropagation: () => {},+ dataTransfer: {types: ['Files'], files: zip}+ });++ wrapper.setProps({+ children: (+ <IntlWrapper>+ <FileUpload+ onFileUpload={() => {}}+ fileExtensions={['zip', 'shp']}+ fileLoading={false}+ fileLoadingProgress={{}}+ stagedToAdd={[{info: {label: 'places.zip', format: 'geojson'}}]}+ />+ </IntlWrapper>+ )+ });++ const listText = wrapper.find('.upload-file-list').text();+ t.ok(listText.includes('places.zip'), 'a single shapefile zip keeps the archive name');+ t.equal(+ wrapper.find('.upload-file-list').find('Checkbox').length,+ 1,+ 'the archive is one dataset'+ );++ t.end();+});+test('Components -> FileUpload remote URL is staged without downloading', t => {const onFileUpload = sinon.spy();const onAddRemoteDataset = sinon.spy();diff --git a/test/node/processors/loader-registry-test.js b/test/node/processors/loader-registry-test.jsindex 096acd8ae..bd24c7048 100644--- a/test/node/processors/loader-registry-test.js+++ b/test/node/processors/loader-registry-test.js@@ -333,6 +333,7 @@ test('#loader-registry -> acceptedFileFormats defaults to all formats', t => {'GPX','TCX','Shapefile',+ 'Zip','Excel','FlatGeobuf'],@@ -350,6 +351,7 @@ test('#loader-registry -> acceptedFileFormats restricts loaders and UI lists', at.notOk(isKeplerFileFormatAccepted('kml'), 'kml is rejected');t.notOk(isKeplerFileFormatAccepted('parquet'), 'parquet is rejected');t.notOk(isKeplerFileFormatAccepted('shp'), 'shp is rejected');+ t.ok(isKeplerFileFormatAccepted('zip'), 'a zip of the allowed formats stays accepted');t.notOk(isKeplerFileFormatAccepted('xlsx'), 'xlsx is rejected');const ids = getAcceptedKeplerLoaderEntries().map(entry => entry.id);@@ -364,12 +366,12 @@ test('#loader-registry -> acceptedFileFormats restricts loaders and UI lists', at.deepEqual(getFileExtensions({loaders: []}),- ['csv', 'tsv', 'dsv', 'json', 'geojson'],+ ['csv', 'tsv', 'dsv', 'json', 'geojson', 'zip'],'file picker extensions follow the allowlist');t.deepEqual(getFileFormatNames({loaders: []}),- ['CSV', 'Json', 'GeoJSON'],+ ['CSV', 'Json', 'GeoJSON', 'Zip'],'file picker labels follow the allowlist');t.deepEqual(diff --git a/test/node/processors/shapefile-files-test.js b/test/node/processors/shapefile-files-test.jsindex 1930af135..d5f400ec6 100644--- a/test/node/processors/shapefile-files-test.js+++ b/test/node/processors/shapefile-files-test.js@@ -4,6 +4,7 @@import test from 'tape';import {crc32} from 'zlib';import {+ expandZipArchive,getFilesToParse,getDroppedFileExtension,isShapefileSidecarFileName,@@ -11,6 +12,7 @@ import {readFileInBatches,unzipShapefileArchive} from '@kepler.gl/processors';+import {initApplicationConfig} from '@kepler.gl/utils';function createPointShapefileBuffers({x = -122.4, y = 37.8, name = 'alpha'} = {}) {const shp = new ArrayBuffer(128);@@ -215,3 +217,156 @@ test('#shapefile-files -> shapefile zip unpacks before parse', async t => {t.equal(processed[0].data.rows.length, 1, 'should keep the unzipped shapefile feature');t.end();});++test('#shapefile-files -> expandZipArchive keeps a single shapefile archive', async t => {+ const {shp, shx, dbf} = createPointShapefileBuffers();+ const zip = createStoredZip([+ {name: 'places.shp', data: Buffer.from(shp)},+ {name: 'places.shx', data: Buffer.from(shx)},+ {name: 'places.dbf', data: Buffer.from(dbf)},+ {name: 'readme.txt', data: Buffer.from('notes')},+ {name: 'nested/ignore.shp', data: Buffer.from(shp)}+ ]);+ const expansion = await expandZipArchive(new File([zip], 'places.zip'));++ t.equal(expansion.kind, 'shapefile', 'one shapefile plus sidecars stays one archive');+ if (expansion.kind !== 'shapefile') {+ t.end();+ return;+ }+ t.equal(expansion.shapefile.name, 'places.shp');+ t.deepEqual(+ expansion.companions.map(file => file.name).sort(),+ ['places.dbf', 'places.shx'],+ 'readme and nested files are not companions'+ );+ t.end();+});++test('#shapefile-files -> expandZipArchive splits mixed files into datasets', async t => {+ const csv = 'name,lat,lng\nalpha,37.8,-122.4\n';+ const zip = createStoredZip([+ {name: 'places.csv', data: Buffer.from(csv)},+ {+ name: 'neighborhoods.geojson',+ data: Buffer.from('{"type":"FeatureCollection","features":[]}')+ },+ {name: 'nested/ignore.csv', data: Buffer.from('a\n1\n')},+ {name: 'readme.txt', data: Buffer.from('hi')},+ {name: 'inner.zip', data: Buffer.from('not-a-zip')}+ ]);+ const expansion = await expandZipArchive(new File([zip], 'bundle.zip'));++ t.equal(expansion.kind, 'datasets');+ if (expansion.kind !== 'datasets') {+ t.end();+ return;+ }+ t.deepEqual(+ expansion.datasets.map(file => file.name),+ ['places.csv', 'neighborhoods.geojson'],+ 'each supported top-level file becomes a dataset'+ );+ t.deepEqual(expansion.companions, [], 'a csv zip has no shapefile sidecars');++ const batch = await readLastBatch(expansion.datasets[0]);+ const processed = await processFileData({content: batch, fileCache: []});+ t.equal(processed[0].info.label, 'places.csv', 'the dataset label is the file inside the zip');+ t.equal(processed[0].data.rows.length, 1, 'should parse the unzipped csv');+ t.end();+});++test('#shapefile-files -> expandZipArchive loads every shapefile in the zip', async t => {+ const alpha = createPointShapefileBuffers({name: 'alpha'});+ const beta = createPointShapefileBuffers({name: 'beta', x: -122.5, y: 37.9});+ const zip = createStoredZip([+ {name: 'alpha.shp', data: Buffer.from(alpha.shp)},+ {name: 'alpha.shx', data: Buffer.from(alpha.shx)},+ {name: 'alpha.dbf', data: Buffer.from(alpha.dbf)},+ {name: 'beta.shp', data: Buffer.from(beta.shp)},+ {name: 'beta.shx', data: Buffer.from(beta.shx)},+ {name: 'beta.dbf', data: Buffer.from(beta.dbf)}+ ]);+ const expansion = await expandZipArchive(new File([zip], 'places.zip'));++ t.equal(expansion.kind, 'datasets', 'several shapefiles are separate datasets');+ if (expansion.kind !== 'datasets') {+ t.end();+ return;+ }+ t.deepEqual(+ expansion.datasets.map(file => file.name),+ ['alpha.shp', 'beta.shp']+ );+ const companions = [...expansion.datasets, ...expansion.companions];+ const names = [];+ for (const file of expansion.datasets) {+ const batch = await readLastBatch(file, companions);+ const processed = await processFileData({content: batch, fileCache: []});+ const nameField = processed[0].data.fields.findIndex(field => field.name === 'NAME');+ names.push(String(processed[0].data.rows[0][nameField]).trim());+ t.equal(processed[0].info.label, file.name, 'each shapefile keeps its own name');+ }+ t.deepEqual(names, ['alpha', 'beta'], 'each shapefile reads its own DBF');+ t.end();+});++test('#shapefile-files -> expandZipArchive rejects a zip with nothing supported', async t => {+ const zip = createStoredZip([+ {name: 'readme.txt', data: Buffer.from('hi')},+ {name: 'nested/places.csv', data: Buffer.from('a\n1\n')}+ ]);+ try {+ await expandZipArchive(new File([zip], 'notes.zip'));+ t.fail('should reject a zip with no supported dataset');+ } catch (error) {+ t.equal(error.message, 'Zip archive does not contain a supported dataset');+ }+ t.end();+});++test('#shapefile-files -> expandZipArchive keeps custom loader extensions', async t => {+ initApplicationConfig({acceptedFileFormats: ['csv']});+ try {+ const zip = createStoredZip([+ {name: 'track.gpx', data: Buffer.from('<gpx></gpx>')},+ {name: 'notes.custom', data: Buffer.from('x')}+ ]);+ const expansion = await expandZipArchive(new File([zip], 'bundle.zip'), ['custom']);+ t.equal(expansion.kind, 'datasets');+ if (expansion.kind === 'datasets') {+ t.deepEqual(+ expansion.datasets.map(file => file.name),+ ['notes.custom'],+ 'custom extensions stay even when that built-in format is not allowed'+ );+ }+ } finally {+ initApplicationConfig({acceptedFileFormats: null});+ }+ t.end();+});++test('#shapefile-files -> expandZipArchive honors accepted file formats', async t => {+ initApplicationConfig({acceptedFileFormats: ['csv']});+ try {+ const {shp} = createPointShapefileBuffers();+ const zip = createStoredZip([+ {name: 'places.csv', data: Buffer.from('name\nalpha\n')},+ {name: 'places.geojson', data: Buffer.from('{"type":"FeatureCollection","features":[]}')},+ {name: 'places.shp', data: Buffer.from(shp)}+ ]);+ const expansion = await expandZipArchive(new File([zip], 'bundle.zip'));+ t.equal(expansion.kind, 'datasets');+ if (expansion.kind === 'datasets') {+ t.deepEqual(+ expansion.datasets.map(file => file.name),+ ['places.csv'],+ 'formats outside the allowlist stay in the zip'+ );+ }+ } finally {+ initApplicationConfig({acceptedFileFormats: null});+ }+ t.end();+});diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex 01e13703f..ea581ac42 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -7092,6 +7092,126 @@ test('#visStateReducer -> LOAD_FILES', async t => {t.end();});+test('#visStateReducer -> LOAD_FILES expands a mixed zip into datasets', t => {+ drainTasksForTesting();++ const initialState = CloneDeep(InitialState).visState;+ const zipFile = {type: 'application/zip', name: 'bundle.zip'};+ const extraFile = {type: 'text/csv', name: 'extra.csv'};+ const nextState = reducer(initialState, VisStateActions.loadFiles([zipFile, extraFile]));+ const [expandTask] = drainTasksForTesting();++ t.equal(expandTask.type, 'EXPAND_ZIP_TASK', 'a zip is unpacked before it is parsed');+ t.equal(expandTask.payload.file, zipFile, 'the expand task receives the zip');+ t.deepEqual(expandTask.payload.extensions, [], 'built-in formats need no extra extensions');+ t.deepEqual(+ nextState.fileLoading.filesToLoad.map(file => file.name),+ ['extra.csv'],+ 'the zip leaves the queue while it is unpacked'+ );++ const places = {type: 'text/csv', name: 'places.csv'};+ const neighborhoods = {type: 'application/geo+json', name: 'neighborhoods.geojson'};+ const expanded = reducer(+ nextState,+ succeedTaskInTest(expandTask, {+ kind: 'datasets',+ datasets: [places, neighborhoods],+ companions: []+ })+ );+ const [loadTask] = drainTasksForTesting();++ t.equal(loadTask.type, 'LOAD_FILE_TASK', 'the first unzipped dataset starts loading');+ t.equal(loadTask.payload.file, places);+ t.equal(loadTask.payload.fileName, undefined, 'mixed members keep their own file names');+ t.deepEqual(+ expanded.fileLoading.filesToLoad.map(file => file.name),+ ['neighborhoods.geojson', 'extra.csv'],+ 'remaining members stay queued after the file that was dropped beside the zip'+ );+ t.equal(+ expanded.fileLoadingProgress['bundle.zip'],+ undefined,+ 'the zip progress row is replaced'+ );+ t.equal(expanded.fileLoadingProgress['places.csv'].message, 'loading...');+ t.ok(expanded.fileLoadingProgress['neighborhoods.geojson']);+ t.ok(expanded.fileLoadingProgress['extra.csv']);+ t.end();+});++test('#visStateReducer -> LOAD_FILES keeps a single shapefile zip name', t => {+ drainTasksForTesting();++ const initialState = CloneDeep(InitialState).visState;+ const zipFile = {type: 'application/zip', name: 'places.zip'};+ const nextState = reducer(initialState, VisStateActions.loadFiles([zipFile]));+ const [expandTask] = drainTasksForTesting();+ const shapefile = {name: 'places.shp'};+ const dbf = {name: 'places.dbf'};+ const expanded = reducer(+ nextState,+ succeedTaskInTest(expandTask, {+ kind: 'shapefile',+ shapefile,+ companions: [dbf]+ })+ );+ const [loadTask] = drainTasksForTesting();++ t.equal(loadTask.type, 'LOAD_FILE_TASK');+ t.equal(loadTask.payload.file, shapefile, 'the shapefile inside the zip is parsed');+ t.equal(+ loadTask.payload.fileName,+ 'places.zip',+ 'progress and the dataset label stay the zip name'+ );+ t.deepEqual(+ loadTask.payload.companionFiles.map(file => file.name),+ ['places.shp', 'places.dbf', 'places.zip'],+ 'sidecars are available while the shapefile parses'+ );+ t.equal(expanded.fileLoading.filesToLoad.length, 0);+ t.equal(expanded.fileLoadingProgress['places.zip'].message, 'loading...');+ t.equal(expanded.fileLoadingProgress['places.shp'], undefined);+ t.end();+});++test('#visStateReducer -> LOAD_FILES passes custom loader extensions into a zip', t => {+ drainTasksForTesting();++ const initialState = {+ ...CloneDeep(InitialState).visState,+ loaders: [{extensions: ['.custom']}]+ };+ const zipFile = {type: 'application/zip', name: 'bundle.zip'};+ reducer(initialState, VisStateActions.loadFiles([zipFile]));+ const [expandTask] = drainTasksForTesting();++ t.deepEqual(+ expandTask.payload,+ {file: zipFile, extensions: ['custom']},+ 'custom loader extensions are unpacked with the zip'+ );+ t.end();+});++test('#visStateReducer -> LOAD_FILES ignores a stale zip error', t => {+ drainTasksForTesting();++ const initialState = CloneDeep(InitialState).visState;+ const first = reducer(initialState, VisStateActions.loadFiles([{name: 'first.zip'}]));+ const [expandTask] = drainTasksForTesting();+ const second = reducer(first, VisStateActions.loadFiles([{name: 'second.csv'}]));+ drainTasksForTesting();++ const errored = reducer(second, errorTaskInTest(expandTask, new Error('bad zip')));+ t.equal(errored, second, 'a zip error from the previous load does not touch the new one');+ t.equal(drainTasksForTesting().length, 0, 'the stale error does not advance the new queue');+ t.end();+});+test('#visStateReducer -> LOAD_FILES autoCreateLayers option', t => {drainTasksForTesting();