Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[Demo] Add GeoArrow sample dataset (#2483)

Xun Lif33b09f8b3d8
diff --git a/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md b/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md
index 7dcc81e18..2d688dc0c 100644
--- a/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md
+++ b/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md
@@ -67,7 +67,7 @@ kepler.gl will auto detect layer, if the column names follows certain naming con
|**Polygon**| A column content contains `geojson` data types. Acceptable formats include [Well-Known Text](http://www.postgis.net/docs/ST_AsText.html) e.g. `POLYGON ((-74.158 40.835, -74.148 40.830, -74.151 40.832, -74.158 40.835))` and [GeoJSON Geometry](https://tools.ietf.org/html/rfc7946#appendix-A). e.g. `{"type":"LineString","coordinates":[[100.0, 0.0],[101.0, 1.0]]}`
#### 3. Embed Geometries in CSV
-Geometries (Polygons, Points, LindStrings etc) can be embedded into CSV as a `GeoJSON` or `WKT` formatted string.
+Geometries (Polygons, Points, LindStrings etc) can be embedded into CSV as a `GeoJSON` or `WKT` formatted string.
##### `GeoJSON` String
Use the geometry of a Feature, which includes type and coordinates. It should be a JSON formatted string, with the `"` corrected escaped. More info on [String escape in csv](https://gpdb.docs.pivotal.io/43250/admin_guide/load/topics/g-escaping-in-csv-formatted-files.html)
@@ -79,7 +79,7 @@ id,geometry
```
##### `WKT`String
-[The Well-Known Text (WKT)](https://dev.mysql.com/doc/refman/5.7/en/gis-data-formats.html#gis-wkt-format) representation of geometry values is designed for exchanging geometry data in ASCII form.
+[The Well-Known Text (WKT)](https://dev.mysql.com/doc/refman/5.7/en/gis-data-formats.html#gis-wkt-format) representation of geometry values is designed for exchanging geometry data in ASCII form.
Example data.csv with WKT
```txt
@@ -226,4 +226,4 @@ To add additional datasets to your map:
[Back to table of contents](../README.md)
-[kepler.gl-data-github]: https://github.com/uber-web/kepler.gl-data
+[kepler.gl-data-github]: https://github.com/keplergl/kepler.gl-data
diff --git a/examples/demo-app/package.json b/examples/demo-app/package.json
index 0eb821902..2a04a507a 100644
--- a/examples/demo-app/package.json
+++ b/examples/demo-app/package.json
@@ -18,6 +18,10 @@
"@kepler.gl/processors": "^3.0.0-alpha.1",
"@kepler.gl/reducers": "^3.0.0-alpha.1",
"@kepler.gl/styles": "^3.0.0-alpha.1",
+ "@loaders.gl/arrow": "^4.1.0-alpha.4",
+ "@loaders.gl/core": "^4.1.0-alpha.4",
+ "@loaders.gl/csv": "^4.1.0-alpha.4",
+ "@loaders.gl/json": "^4.1.0-alpha.4",
"d3-request": "^1.0.6",
"dropbox": "^4.0.12",
"global": "^4.3.0",
diff --git a/examples/demo-app/src/actions.js b/examples/demo-app/src/actions.js
index b6bd7548c..62e0e149a 100644
--- a/examples/demo-app/src/actions.js
+++ b/examples/demo-app/src/actions.js
@@ -21,6 +21,10 @@
import {push} from 'react-router-redux';
import {request, text as requestText, json as requestJson} from 'd3-request';
import {loadFiles, toggleModal} from '@kepler.gl/actions';
+import {load} from '@loaders.gl/core';
+import {CSVLoader} from '@loaders.gl/csv';
+import {ArrowLoader} from '@loaders.gl/arrow';
+import {_GeoJSONLoader as GeoJSONLoader} from '@loaders.gl/json';
import {
LOADING_SAMPLE_ERROR_MESSAGE,
@@ -183,8 +187,8 @@ function loadRemoteRawData(url) {
/**
*
* @param {Object} options
- * @param {string} [options.dataUrl] the URL to fetch data from, e.g. https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/earthquakes/data.csv
- * @param {string} [options.configUrl] the URL string to fetch kepler config from, e.g. https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/earthquakes/config.json
+ * @param {string} [options.dataUrl] the URL to fetch data from, e.g. https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/earthquakes/data.csv
+ * @param {string} [options.configUrl] the URL string to fetch kepler config from, e.g. https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/earthquakes/config.json
* @param {string} [options.id] the id used as dataset unique identifier, e.g. earthquakes
* @param {string} [options.label] the label used to describe the new dataset, e.g. California Earthquakes
* @param {string} [options.queryType] the type of query to execute to load data/config, e.g. sample
@@ -290,19 +294,19 @@ function loadRemoteData(url) {
}
// Load data
- return new Promise((resolve, reject) => {
- requestMethod(url, (error, result) => {
- if (error) {
- reject(error);
- return;
- }
- const responseError = detectResponseError(result);
- if (responseError) {
- reject(responseError);
- return;
- }
- resolve(result);
- });
+ return new Promise(resolve => {
+ const loaders = [CSVLoader, ArrowLoader, GeoJSONLoader];
+ const loadOptions = {
+ arrow: {
+ shape: 'arrow-table'
+ },
+ csv: {
+ shape: 'object-row-table'
+ },
+ metadata: true
+ };
+ const data = load(url, loaders, loadOptions);
+ resolve(data);
});
}
diff --git a/examples/demo-app/src/components/map-control/map-control.js b/examples/demo-app/src/components/map-control/map-control.js
index e38bf0e38..fdd6b20c3 100644
--- a/examples/demo-app/src/components/map-control/map-control.js
+++ b/examples/demo-app/src/components/map-control/map-control.js
@@ -113,8 +113,8 @@ const LinkRenderer = props => {
);
};
-// convert https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/nyctrips/config.json
-// to https://github.com/uber-web/kepler.gl-data/blob/master/movement_pittsburgh/config.json
+// convert https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/nyctrips/config.json
+// to https://github.com/keplergl/kepler.gl-data/blob/master/movement_pittsburgh/config.json
function getURL(url) {
return url
? url
diff --git a/examples/demo-app/src/constants/default-settings.js b/examples/demo-app/src/constants/default-settings.js
index 8b4e0ebb9..1f22da82c 100644
--- a/examples/demo-app/src/constants/default-settings.js
+++ b/examples/demo-app/src/constants/default-settings.js
@@ -23,7 +23,7 @@ and segments both use queryRunner */
import keyMirror from 'keymirror';
export const ASSETS_URL = 'https://d1a3f4spazzrp4.cloudfront.net/kepler.gl/';
-export const DATA_URL = 'https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/';
+export const DATA_URL = 'https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/';
export const MAP_URI = 'demo/map?mapUrl=';
/*
* If you want to add more samples, feel free to edit the json file on github kepler.gl data repo
diff --git a/examples/demo-app/src/reducers/index.js b/examples/demo-app/src/reducers/index.js
index a7673ea93..b572ab13c 100644
--- a/examples/demo-app/src/reducers/index.js
+++ b/examples/demo-app/src/reducers/index.js
@@ -22,7 +22,7 @@ import {combineReducers} from 'redux';
import {handleActions} from 'redux-actions';
import keplerGlReducer, {combinedUpdaters, uiStateUpdaters} from '@kepler.gl/reducers';
-import {processGeojson, processCsvData} from '@kepler.gl/processors';
+import {processGeojson, processRowObject, processArrowTable} from '@kepler.gl/processors';
import KeplerGlSchema from '@kepler.gl/schemas';
import {EXPORT_MAP_FORMATS} from '@kepler.gl/constants';
@@ -107,10 +107,12 @@ export const loadRemoteResourceSuccess = (state, action) => {
// TODO: replace generate with a different function
const datasetId = action.options.id || generateHashId(6);
const {dataUrl} = action.options;
- let processorMethod = processCsvData;
+ let processorMethod = processRowObject;
// TODO: create helper to determine file ext eligibility
if (dataUrl.includes('.json') || dataUrl.includes('.geojson')) {
processorMethod = processGeojson;
+ } else if (dataUrl.includes('.arrow')) {
+ processorMethod = processArrowTable;
}
const datasets = {
diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts
index d95bc4b7c..f38dee398 100644
--- a/src/layers/src/geojson-layer/geojson-layer.ts
+++ b/src/layers/src/geojson-layer/geojson-layer.ts
@@ -359,7 +359,7 @@ export default class GeoJsonLayer extends Layer {
return null;
}
- calculateDataAttribute({ dataContainer, filteredIndex }, getPosition) {
+ calculateDataAttribute({dataContainer, filteredIndex}, getPosition) {
if (dataContainer instanceof ArrowDataContainer) {
// filter geojson/arrow table by values and make a partial copy of the raw table are expensive
// so we will use filteredIndex to create an attribute e.g. filteredIndex [0|1] for GPU filtering
@@ -379,7 +379,7 @@ export default class GeoJsonLayer extends Layer {
}
this.filteredIndexTrigger = filteredIndex;
}
- // for arrow, always return full dataToFeature instead of a filtered one, so there is no need to update attributes in GPU
+ // for arrow, always return full dataToFeature instead of a filtered one, so there is no need to update attributes in GPU
return this.dataToFeature;
}
@@ -427,13 +427,15 @@ export default class GeoJsonLayer extends Layer {
if (dataContainer instanceof ArrowDataContainer) {
// update the latest batch/chunk of geoarrow data when loading data incrementally
if (this.dataToFeature.length < dataContainer.numChunks()) {
+ // for incrementally loading data, we only load and render the latest batch; otherwise, we will load and render all batches
+ const isIncrementalLoad = dataContainer.numChunks() - this.dataToFeature.length === 1;
const {dataToFeature, bounds, fixedRadius, featureTypes} = getGeojsonLayerMetaFromArrow({
dataContainer,
getGeoColumn,
getGeoField,
- chunkIndex: this.dataToFeature.length
+ ...(isIncrementalLoad ? {chunkIndex: this.dataToFeature.length} : null)
});
- this.updateMeta({ bounds, fixedRadius, featureTypes });
+ this.updateMeta({bounds, fixedRadius, featureTypes});
this.dataToFeature = [...this.dataToFeature, ...dataToFeature];
}
} else {
@@ -443,7 +445,7 @@ export default class GeoJsonLayer extends Layer {
getFeature
});
this.dataToFeature = dataToFeature;
- this.updateMeta({ bounds, fixedRadius, featureTypes });
+ this.updateMeta({bounds, fixedRadius, featureTypes});
}
}
}
diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.ts
index 311df6946..176370b8b 100644
--- a/src/processors/src/data-processor.ts
+++ b/src/processors/src/data-processor.ts
@@ -22,6 +22,7 @@ import * as arrow from 'apache-arrow';
import {csvParseRows} from 'd3-dsv';
import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer';
import normalize from '@mapbox/geojson-normalize';
+import {ArrowTable} from '@loaders.gl/schema';
import {ALL_FIELD_TYPES, DATASET_FORMATS, GUIDES_FILE_FORMAT_DOC} from '@kepler.gl/constants';
import {ProcessorResult, Field} from '@kepler.gl/types';
import {
@@ -393,12 +394,22 @@ export function processKeplerglDataset(
}
/**
- * Parse a arrow table with geometry columns and return a dataset
+ * Parse arrow table and return a dataset
+ *
+ * @param arrowTable ArrowTable to parse, see loaders.gl/schema
+ * @returns dataset containing `fields` and `rows` or null
+ */
+export function processArrowTable(arrowTable: ArrowTable): ProcessorResult | null {
+ return processArrowBatches(arrowTable.data.batches);
+}
+
+/**
+ * Parse arrow batches returned from parseInBatches()
*
* @param arrowTable the arrow table to parse
* @returns dataset containing `fields` and `rows` or null
*/
-export function processArrowTable(arrowBatches: arrow.RecordBatch[]): ProcessorResult | null {
+export function processArrowBatches(arrowBatches: arrow.RecordBatch[]): ProcessorResult | null {
if (arrowBatches.length === 0) {
return null;
}
@@ -442,6 +453,7 @@ export const Processors: {
processGeojson: typeof processGeojson;
processCsvData: typeof processCsvData;
processArrowTable: typeof processArrowTable;
+ processArrowBatches: typeof processArrowBatches;
processRowObject: typeof processRowObject;
processKeplerglJSON: typeof processKeplerglJSON;
processKeplerglDataset: typeof processKeplerglDataset;
@@ -452,6 +464,7 @@ export const Processors: {
processGeojson,
processCsvData,
processArrowTable,
+ processArrowBatches,
processRowObject,
processKeplerglJSON,
processKeplerglDataset,
diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.ts
index 72f327583..15652c181 100644
--- a/src/processors/src/file-handler.ts
+++ b/src/processors/src/file-handler.ts
@@ -24,7 +24,7 @@ import {JSONLoader, _JSONPath} from '@loaders.gl/json';
import {CSVLoader} from '@loaders.gl/csv';
import {ArrowLoader} from '@loaders.gl/arrow';
import {
- processArrowTable,
+ processArrowBatches,
processGeojson,
processKeplerglJSON,
processRowObject
@@ -225,7 +225,7 @@ export function processFileData({
if (isArrowData(data)) {
format = DATASET_FORMATS.arrow;
- processor = processArrowTable;
+ processor = processArrowBatches;
} else if (isKeplerGlMap(data)) {
format = DATASET_FORMATS.keplergl;
processor = processKeplerglJSON;