Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[Demo] Add GeoArrow sample dataset (#2483)
Xun Lif33b09f8b3d8
diff --git a/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md b/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.mdindex 7dcc81e18..2d688dc0c 100644--- a/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md+++ b/docs/user-guides/b-kepler-gl-workflow/a-add-data-to-the-map.md@@ -67,7 +67,7 @@ kepler.gl will auto detect layer, if the column names follows certain naming con|**Polygon**| A column content contains `geojson` data types. Acceptable formats include [Well-Known Text](http://www.postgis.net/docs/ST_AsText.html) e.g. `POLYGON ((-74.158 40.835, -74.148 40.830, -74.151 40.832, -74.158 40.835))` and [GeoJSON Geometry](https://tools.ietf.org/html/rfc7946#appendix-A). e.g. `{"type":"LineString","coordinates":[[100.0, 0.0],[101.0, 1.0]]}`#### 3. Embed Geometries in CSV-Geometries (Polygons, Points, LindStrings etc) can be embedded into CSV as a `GeoJSON` or `WKT` formatted string.+Geometries (Polygons, Points, LindStrings etc) can be embedded into CSV as a `GeoJSON` or `WKT` formatted string.##### `GeoJSON` StringUse the geometry of a Feature, which includes type and coordinates. It should be a JSON formatted string, with the `"` corrected escaped. More info on [String escape in csv](https://gpdb.docs.pivotal.io/43250/admin_guide/load/topics/g-escaping-in-csv-formatted-files.html)@@ -79,7 +79,7 @@ id,geometry```##### `WKT`String-[The Well-Known Text (WKT)](https://dev.mysql.com/doc/refman/5.7/en/gis-data-formats.html#gis-wkt-format) representation of geometry values is designed for exchanging geometry data in ASCII form.+[The Well-Known Text (WKT)](https://dev.mysql.com/doc/refman/5.7/en/gis-data-formats.html#gis-wkt-format) representation of geometry values is designed for exchanging geometry data in ASCII form.Example data.csv with WKT```txt@@ -226,4 +226,4 @@ To add additional datasets to your map:[Back to table of contents](../README.md)-[kepler.gl-data-github]: https://github.com/uber-web/kepler.gl-data+[kepler.gl-data-github]: https://github.com/keplergl/kepler.gl-datadiff --git a/examples/demo-app/package.json b/examples/demo-app/package.jsonindex 0eb821902..2a04a507a 100644--- a/examples/demo-app/package.json+++ b/examples/demo-app/package.json@@ -18,6 +18,10 @@"@kepler.gl/processors": "^3.0.0-alpha.1","@kepler.gl/reducers": "^3.0.0-alpha.1","@kepler.gl/styles": "^3.0.0-alpha.1",+ "@loaders.gl/arrow": "^4.1.0-alpha.4",+ "@loaders.gl/core": "^4.1.0-alpha.4",+ "@loaders.gl/csv": "^4.1.0-alpha.4",+ "@loaders.gl/json": "^4.1.0-alpha.4","d3-request": "^1.0.6","dropbox": "^4.0.12","global": "^4.3.0",diff --git a/examples/demo-app/src/actions.js b/examples/demo-app/src/actions.jsindex b6bd7548c..62e0e149a 100644--- a/examples/demo-app/src/actions.js+++ b/examples/demo-app/src/actions.js@@ -21,6 +21,10 @@import {push} from 'react-router-redux';import {request, text as requestText, json as requestJson} from 'd3-request';import {loadFiles, toggleModal} from '@kepler.gl/actions';+import {load} from '@loaders.gl/core';+import {CSVLoader} from '@loaders.gl/csv';+import {ArrowLoader} from '@loaders.gl/arrow';+import {_GeoJSONLoader as GeoJSONLoader} from '@loaders.gl/json';import {LOADING_SAMPLE_ERROR_MESSAGE,@@ -183,8 +187,8 @@ function loadRemoteRawData(url) {/**** @param {Object} options- * @param {string} [options.dataUrl] the URL to fetch data from, e.g. https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/earthquakes/data.csv- * @param {string} [options.configUrl] the URL string to fetch kepler config from, e.g. https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/earthquakes/config.json+ * @param {string} [options.dataUrl] the URL to fetch data from, e.g. https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/earthquakes/data.csv+ * @param {string} [options.configUrl] the URL string to fetch kepler config from, e.g. https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/earthquakes/config.json* @param {string} [options.id] the id used as dataset unique identifier, e.g. earthquakes* @param {string} [options.label] the label used to describe the new dataset, e.g. California Earthquakes* @param {string} [options.queryType] the type of query to execute to load data/config, e.g. sample@@ -290,19 +294,19 @@ function loadRemoteData(url) {}// Load data- return new Promise((resolve, reject) => {- requestMethod(url, (error, result) => {- if (error) {- reject(error);- return;- }- const responseError = detectResponseError(result);- if (responseError) {- reject(responseError);- return;- }- resolve(result);- });+ return new Promise(resolve => {+ const loaders = [CSVLoader, ArrowLoader, GeoJSONLoader];+ const loadOptions = {+ arrow: {+ shape: 'arrow-table'+ },+ csv: {+ shape: 'object-row-table'+ },+ metadata: true+ };+ const data = load(url, loaders, loadOptions);+ resolve(data);});}diff --git a/examples/demo-app/src/components/map-control/map-control.js b/examples/demo-app/src/components/map-control/map-control.jsindex e38bf0e38..fdd6b20c3 100644--- a/examples/demo-app/src/components/map-control/map-control.js+++ b/examples/demo-app/src/components/map-control/map-control.js@@ -113,8 +113,8 @@ const LinkRenderer = props => {);};-// convert https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/nyctrips/config.json-// to https://github.com/uber-web/kepler.gl-data/blob/master/movement_pittsburgh/config.json+// convert https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/nyctrips/config.json+// to https://github.com/keplergl/kepler.gl-data/blob/master/movement_pittsburgh/config.jsonfunction getURL(url) {return url? urldiff --git a/examples/demo-app/src/constants/default-settings.js b/examples/demo-app/src/constants/default-settings.jsindex 8b4e0ebb9..1f22da82c 100644--- a/examples/demo-app/src/constants/default-settings.js+++ b/examples/demo-app/src/constants/default-settings.js@@ -23,7 +23,7 @@ and segments both use queryRunner */import keyMirror from 'keymirror';export const ASSETS_URL = 'https://d1a3f4spazzrp4.cloudfront.net/kepler.gl/';-export const DATA_URL = 'https://raw.githubusercontent.com/uber-web/kepler.gl-data/master/';+export const DATA_URL = 'https://raw.githubusercontent.com/keplergl/kepler.gl-data/master/';export const MAP_URI = 'demo/map?mapUrl=';/** If you want to add more samples, feel free to edit the json file on github kepler.gl data repodiff --git a/examples/demo-app/src/reducers/index.js b/examples/demo-app/src/reducers/index.jsindex a7673ea93..b572ab13c 100644--- a/examples/demo-app/src/reducers/index.js+++ b/examples/demo-app/src/reducers/index.js@@ -22,7 +22,7 @@ import {combineReducers} from 'redux';import {handleActions} from 'redux-actions';import keplerGlReducer, {combinedUpdaters, uiStateUpdaters} from '@kepler.gl/reducers';-import {processGeojson, processCsvData} from '@kepler.gl/processors';+import {processGeojson, processRowObject, processArrowTable} from '@kepler.gl/processors';import KeplerGlSchema from '@kepler.gl/schemas';import {EXPORT_MAP_FORMATS} from '@kepler.gl/constants';@@ -107,10 +107,12 @@ export const loadRemoteResourceSuccess = (state, action) => {// TODO: replace generate with a different functionconst datasetId = action.options.id || generateHashId(6);const {dataUrl} = action.options;- let processorMethod = processCsvData;+ let processorMethod = processRowObject;// TODO: create helper to determine file ext eligibilityif (dataUrl.includes('.json') || dataUrl.includes('.geojson')) {processorMethod = processGeojson;+ } else if (dataUrl.includes('.arrow')) {+ processorMethod = processArrowTable;}const datasets = {diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.tsindex d95bc4b7c..f38dee398 100644--- a/src/layers/src/geojson-layer/geojson-layer.ts+++ b/src/layers/src/geojson-layer/geojson-layer.ts@@ -359,7 +359,7 @@ export default class GeoJsonLayer extends Layer {return null;}- calculateDataAttribute({ dataContainer, filteredIndex }, getPosition) {+ calculateDataAttribute({dataContainer, filteredIndex}, getPosition) {if (dataContainer instanceof ArrowDataContainer) {// filter geojson/arrow table by values and make a partial copy of the raw table are expensive// so we will use filteredIndex to create an attribute e.g. filteredIndex [0|1] for GPU filtering@@ -379,7 +379,7 @@ export default class GeoJsonLayer extends Layer {}this.filteredIndexTrigger = filteredIndex;}- // for arrow, always return full dataToFeature instead of a filtered one, so there is no need to update attributes in GPU+ // for arrow, always return full dataToFeature instead of a filtered one, so there is no need to update attributes in GPUreturn this.dataToFeature;}@@ -427,13 +427,15 @@ export default class GeoJsonLayer extends Layer {if (dataContainer instanceof ArrowDataContainer) {// update the latest batch/chunk of geoarrow data when loading data incrementallyif (this.dataToFeature.length < dataContainer.numChunks()) {+ // for incrementally loading data, we only load and render the latest batch; otherwise, we will load and render all batches+ const isIncrementalLoad = dataContainer.numChunks() - this.dataToFeature.length === 1;const {dataToFeature, bounds, fixedRadius, featureTypes} = getGeojsonLayerMetaFromArrow({dataContainer,getGeoColumn,getGeoField,- chunkIndex: this.dataToFeature.length+ ...(isIncrementalLoad ? {chunkIndex: this.dataToFeature.length} : null)});- this.updateMeta({ bounds, fixedRadius, featureTypes });+ this.updateMeta({bounds, fixedRadius, featureTypes});this.dataToFeature = [...this.dataToFeature, ...dataToFeature];}} else {@@ -443,7 +445,7 @@ export default class GeoJsonLayer extends Layer {getFeature});this.dataToFeature = dataToFeature;- this.updateMeta({ bounds, fixedRadius, featureTypes });+ this.updateMeta({bounds, fixedRadius, featureTypes});}}}diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.tsindex 311df6946..176370b8b 100644--- a/src/processors/src/data-processor.ts+++ b/src/processors/src/data-processor.ts@@ -22,6 +22,7 @@ import * as arrow from 'apache-arrow';import {csvParseRows} from 'd3-dsv';import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer';import normalize from '@mapbox/geojson-normalize';+import {ArrowTable} from '@loaders.gl/schema';import {ALL_FIELD_TYPES, DATASET_FORMATS, GUIDES_FILE_FORMAT_DOC} from '@kepler.gl/constants';import {ProcessorResult, Field} from '@kepler.gl/types';import {@@ -393,12 +394,22 @@ export function processKeplerglDataset(}/**- * Parse a arrow table with geometry columns and return a dataset+ * Parse arrow table and return a dataset+ *+ * @param arrowTable ArrowTable to parse, see loaders.gl/schema+ * @returns dataset containing `fields` and `rows` or null+ */+export function processArrowTable(arrowTable: ArrowTable): ProcessorResult | null {+ return processArrowBatches(arrowTable.data.batches);+}++/**+ * Parse arrow batches returned from parseInBatches()** @param arrowTable the arrow table to parse* @returns dataset containing `fields` and `rows` or null*/-export function processArrowTable(arrowBatches: arrow.RecordBatch[]): ProcessorResult | null {+export function processArrowBatches(arrowBatches: arrow.RecordBatch[]): ProcessorResult | null {if (arrowBatches.length === 0) {return null;}@@ -442,6 +453,7 @@ export const Processors: {processGeojson: typeof processGeojson;processCsvData: typeof processCsvData;processArrowTable: typeof processArrowTable;+ processArrowBatches: typeof processArrowBatches;processRowObject: typeof processRowObject;processKeplerglJSON: typeof processKeplerglJSON;processKeplerglDataset: typeof processKeplerglDataset;@@ -452,6 +464,7 @@ export const Processors: {processGeojson,processCsvData,processArrowTable,+ processArrowBatches,processRowObject,processKeplerglJSON,processKeplerglDataset,diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.tsindex 72f327583..15652c181 100644--- a/src/processors/src/file-handler.ts+++ b/src/processors/src/file-handler.ts@@ -24,7 +24,7 @@ import {JSONLoader, _JSONPath} from '@loaders.gl/json';import {CSVLoader} from '@loaders.gl/csv';import {ArrowLoader} from '@loaders.gl/arrow';import {- processArrowTable,+ processArrowBatches,processGeojson,processKeplerglJSON,processRowObject@@ -225,7 +225,7 @@ export function processFileData({if (isArrowData(data)) {format = DATASET_FORMATS.arrow;- processor = processArrowTable;+ processor = processArrowBatches;} else if (isKeplerGlMap(data)) {format = DATASET_FORMATS.keplergl;processor = processKeplerglJSON;