Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[Docs] Export processors and Add Docs (#421)
Shan Hefaa462a360f9
diff --git a/docs/api-reference/processors/processors.md b/docs/api-reference/processors/processors.mdnew file mode 100644index 000000000..eeb864625--- /dev/null+++ b/docs/api-reference/processors/processors.md@@ -0,0 +1,223 @@+<!-- Generated by documentation.js. Update this documentation by updating the source code. -->++### Table of Contents++- [getFieldsFromData](#getfieldsfromdata)+- [processCsvData](#processcsvdata)+- [processGeojson](#processgeojson)+- [processKeplerglJSON](#processkeplergljson)+- [processRowObject](#processrowobject)++## getFieldsFromData++Analyze field types from data in `string` format, e.g. uploaded csv.+Assign `type`, `tableFieldIndex` and `format` (timestamp only) to each field++### Parameters++- `data` **[Array][16]<[Object][17]>** array of row object+- `fieldOrder` **[Array][16]** array of field names as string++### Examples++```javascript+import {getFieldsFromData} from 'kepler.gl/processors';+const data = [{+ time: '2016-09-17 00:09:55',+ value: '4',+ surge: '1.2',+ isTrip: 'true',+ zeroOnes: '0'+}, {+ time: '2016-09-17 00:30:08',+ value: '3',+ surge: null,+ isTrip: 'false',+ zeroOnes: '1'+}, {+ time: null,+ value: '2',+ surge: '1.3',+ isTrip: null,+ zeroOnes: '1'+}];++const fieldOrder = ['time', 'value', 'surge', 'isTrip', 'zeroOnes'];+const fields = getFieldsFromData(data, fieldOrder);+// fields = [+// {name: 'time', format: 'YYYY-M-D H:m:s', tableFieldIndex: 1, type: 'timestamp'},+// {name: 'value', format: '', tableFieldIndex: 4, type: 'integer'},+// {name: 'surge', format: '', tableFieldIndex: 5, type: 'real'},+// {name: 'isTrip', format: '', tableFieldIndex: 6, type: 'boolean'},+// {name: 'zeroOnes', format: '', tableFieldIndex: 7, type: 'integer'}];+```++Returns **[Array][16]<[Object][17]>** formatted fields++## processCsvData++Process csv data, output a data object with `{fields: [], rows: []}`.+The data object can be wrapped in a `dataset` and pass to [`addDataToMap`][18]++### Parameters++- `rawData` **[string][19]** raw csv string++### Examples++```javascript+import {processCsvData} from 'kepler.gl/processors';++const testData = `gps_data.utc_timestamp,gps_data.lat,gps_data.lng,gps_data.types,epoch,has_result,id,time,begintrip_ts_utc,begintrip_ts_local,date+2016-09-17 00:09:55,29.9900937,31.2590542,driver_analytics,1472688000000,False,1,2016-09-23T00:00:00.000Z,2016-10-01 09:41:39+00:00,2016-10-01 09:41:39+00:00,2016-09-23+2016-09-17 00:10:56,29.9927699,31.2461142,driver_analytics,1472688000000,False,2,2016-09-23T00:00:00.000Z,2016-10-01 09:46:37+00:00,2016-10-01 16:46:37+00:00,2016-09-23+2016-09-17 00:11:56,29.9907261,31.2312742,driver_analytics,1472688000000,False,3,2016-09-23T00:00:00.000Z,,,2016-09-23+2016-09-17 00:12:58,29.9870074,31.2175827,driver_analytics,1472688000000,False,4,2016-09-23T00:00:00.000Z,,,2016-09-23`++const dataset = {+ info: {id: 'test_data', label: 'My Csv'},+ data: processCsvData(testData)+};++dispatch(addDataToMap({+ datasets: [dataset],+ options: {centerMap: true, readOnly: true}+}));+```++Returns **[Object][17]** data object `{fields: [], rows: []}`++## processGeojson++Process GeoJSON [`FeatureCollection`][20],+output a data object with `{fields: [], rows: []}`.+The data object can be wrapped in a `dataset` and pass to [`addDataToMap`][18]++### Parameters++- `rawData` **[Object][17]** raw geojson feature collection++### Examples++```javascript+import {addDataToMap} from 'kepler.gl/actions';+import {processGeojson} from 'kepler.gl/processors';++const geojson = {+ "type" : "FeatureCollection",+ "features" : [{+ "type" : "Feature",+ "properties" : {+ "capacity" : "10",+ "type" : "U-Rack"+ },+ "geometry" : {+ "type" : "Point",+ "coordinates" : [ -71.073283, 42.417500 ]+ }+ }]+};++dispatch(addDataToMap({+ datasets: {+ info: {+ label: 'Sample Taxi Trips in New York City',+ id: 'test_trip_data'+ },+ data: processGeojson(geojson)+ }+}));+```++Returns **[Object][17]** dataset containing `fields` and `rows`++## processKeplerglJSON++Process saved kepler.gl json to be pass to [`addDataToMap`][18].+The json object should contain `datasets` and `config`.++### Parameters++- `rawData` **[Object][17]**+ - `rawData.datasets` **[Array][16]**+ - `rawData.config` **[Object][17]**++### Examples++```javascript+import {addDataToMap} from 'kepler.gl/actions';+import {processKeplerglJSON} from 'kepler.gl/processors';++dispatch(addDataToMap(processKeplerglJSON(keplerGlJson)));+```++Returns **[Object][17]** datasets and config `{datasets: {}, config: {}}`++## processRowObject++Process data where each row is an object, output can be passed to [`addDataToMap`][18]++### Parameters++- `rawData` **[Object][17]** an array of object++### Examples++```javascript+import {addDataToMap} from 'kepler.gl/actions';+import {processRowObject} from 'kepler.gl/processors';++const data = [+ {lat: 31.27, lng: 127.56, value: 3},+ {lat: 31.22, lng: 126.26, value: 1}+];++dispatch(addDataToMap({+ datasets: {+ info: {label: 'My Data', id: 'my_data'},+ data: processRowObject(data)+ }+}));+```++Returns **[Object][17]** dataset containing `fields` and `rows`++[1]: #getfieldsfromdata++[2]: #parameters++[3]: #examples++[4]: #processcsvdata++[5]: #parameters-1++[6]: #examples-1++[7]: #processgeojson++[8]: #parameters-2++[9]: #examples-2++[10]: #processkeplergljson++[11]: #parameters-3++[12]: #examples-3++[13]: #processrowobject++[14]: #parameters-4++[15]: #examples-4++[16]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array++[17]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object++[18]: ../actions/actions.md#adddatatomap++[19]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String++[20]: http://wiki.geojson.org/GeoJSON_draft_version_6#FeatureCollectiondiff --git a/package.json b/package.jsonindex 36afffc2c..352989605 100644--- a/package.json+++ b/package.json@@ -27,7 +27,7 @@"install:web": "yarn install:example && cd website && yarn --ignore-engines","install-and-start": "node ./scripts/install-and-start","fast-test": "npm run test-node && npm run test-browser",- "test-browser": "NODE_ENV=test babel-tape-runner -r ./test/setup-browser-env.js ./test/browser.js",+ "test-browser": "NODE_ENV=test babel-tape-runner -r ./test/setup-browser-env.js ./test/browser.js | tap-spec","test-node": "NODE_ENV=test babel-tape-runner ./test/node.js | tap-spec","test": "npm run lint && npm run test-node && npm run test-browser","cover": "nyc --reporter html --reporter cobertura --reporter json-summary npm test && nyc report",diff --git a/scripts/documentation.js b/scripts/documentation.jsindex 90e6d5b55..2d136766f 100644--- a/scripts/documentation.js+++ b/scripts/documentation.js@@ -76,6 +76,12 @@ const TREE = {{input: 'map-style-updaters.js', output: 'map-style.md', config: {shallow: true}},{input: 'ui-state-updaters.js', output: 'ui-state.md', config: {shallow: true}}]+ },+ {+ path: 'processors',+ children: [+ {input: 'data-processor.js', output: 'processors.md', config: {shallow: true}}+ ]}]}diff --git a/src/processors/data-processor.js b/src/processors/data-processor.jsindex e34d3ada8..c331ccda7 100644--- a/src/processors/data-processor.js+++ b/src/processors/data-processor.js@@ -31,8 +31,32 @@ import KeplerGlSchema from 'schemas';// if any of these value occurs in csv, parse it to null;const CSV_NULLS = ['', 'null', 'NULL', 'Null', 'NaN'];+/**+ * Process csv data, output a data object with `{fields: [], rows: []}`.+ * The data object can be wrapped in a `dataset` and pass to [`addDataToMap`](../actions/actions.md#adddatatomap)+ * @param {string} rawData raw csv string+ * @returns {Object} data object `{fields: [], rows: []}`+ * @public+ * @example+ * import {processCsvData} from 'kepler.gl/processors';+ *+ * const testData = `gps_data.utc_timestamp,gps_data.lat,gps_data.lng,gps_data.types,epoch,has_result,id,time,begintrip_ts_utc,begintrip_ts_local,date+ * 2016-09-17 00:09:55,29.9900937,31.2590542,driver_analytics,1472688000000,False,1,2016-09-23T00:00:00.000Z,2016-10-01 09:41:39+00:00,2016-10-01 09:41:39+00:00,2016-09-23+ * 2016-09-17 00:10:56,29.9927699,31.2461142,driver_analytics,1472688000000,False,2,2016-09-23T00:00:00.000Z,2016-10-01 09:46:37+00:00,2016-10-01 16:46:37+00:00,2016-09-23+ * 2016-09-17 00:11:56,29.9907261,31.2312742,driver_analytics,1472688000000,False,3,2016-09-23T00:00:00.000Z,,,2016-09-23+ * 2016-09-17 00:12:58,29.9870074,31.2175827,driver_analytics,1472688000000,False,4,2016-09-23T00:00:00.000Z,,,2016-09-23`+ *+ * const dataset = {+ * info: {id: 'test_data', label: 'My Csv'},+ * data: processCsvData(testData)+ * };+ *+ * dispatch(addDataToMap({+ * datasets: [dataset],+ * options: {centerMap: true, readOnly: true}+ * }));+ */export function processCsvData(rawData) {-// here we assume the csv file that people uploaded will have first row// as name of the column// TODO: add a alert at upload csv to remind define first row@@ -51,18 +75,29 @@ export function processCsvData(rawData) {const fields = getFieldsFromData(sample, headerRow);- fields.forEach(parseCsvDataByFieldType.bind(null, rows));+ const parsedRows = parseCsvByFields(rows, fields);- return {fields, rows};+ return {fields, rows: parsedRows};}/**- * get fields from csv data+ * Parse rows of csv by analyzed field types. So that `'1'` -> `1`, `'True'` -> `true`+ * @param {Array<Array>} rows+ * @param {Array<Object} fields+ */+export function parseCsvByFields(rows, fields) {+ // Edit rows in place+ fields.forEach(parseCsvRowsByFieldType.bind(null, rows));++ return rows;+}+/**+ * Getting sample data for analyzing field type.*- * @param {array} fields - an array of fields name- * @param {array} allData- * @param {array} sampleCount- * @returns {array} formatted fields+ * @param {Array<string>} fields an array of field names+ * @param {Array<Array>} allData+ * @param {Array} sampleCount+ * @returns {Array} formatted fields*/export function getSampleForTypeAnalyze({fields, allData, sampleCount = 50}) {const total = Math.min(sampleCount, allData.length);@@ -94,6 +129,12 @@ export function getSampleForTypeAnalyze({fields, allData, sampleCount = 50}) {return sample;}+/**+ * Convert falsy value in csv including `'', 'null', 'NULL', 'Null', 'NaN'` to `null`,+ * so that type-analyzer won't detect it as string+ *+ * @param {Array<Array>} rows+ */function cleanUpFalsyCsvValue(rows) {for (let i = 0; i < rows.length; i++) {for (let j = 0; j < rows[i].length; j++) {@@ -110,12 +151,12 @@ function cleanUpFalsyCsvValue(rows) {/*** Process uploaded csv file to parse value by field type*- * @param {array} rows- * @param {object} field- * @param {number} i+ * @param {Array<Array>} rows+ * @param {Object} field+ * @param {Number} i* @returns {void}*/-export function parseCsvDataByFieldType(rows, field, i) {+export function parseCsvRowsByFieldType(rows, field, i) {const unixFormat = ['x', 'X'];rows.forEach(row => {@@ -148,11 +189,45 @@ export function parseCsvDataByFieldType(rows, field, i) {}/**- * get fields from csv data+ * Analyze field types from data in `string` format, e.g. uploaded csv.+ * Assign `type`, `tableFieldIndex` and `format` (timestamp only) to each field+ *+ * @param {Array<Object>} data array of row object+ * @param {Array} fieldOrder array of field names as string+ * @returns {Array<Object>} formatted fields+ * @public+ * @example+ *+ * import {getFieldsFromData} from 'kepler.gl/processors';+ * const data = [{+ * time: '2016-09-17 00:09:55',+ * value: '4',+ * surge: '1.2',+ * isTrip: 'true',+ * zeroOnes: '0'+ * }, {+ * time: '2016-09-17 00:30:08',+ * value: '3',+ * surge: null,+ * isTrip: 'false',+ * zeroOnes: '1'+ * }, {+ * time: null,+ * value: '2',+ * surge: '1.3',+ * isTrip: null,+ * zeroOnes: '1'+ * }];+ *+ * const fieldOrder = ['time', 'value', 'surge', 'isTrip', 'zeroOnes'];+ * const fields = getFieldsFromData(data, fieldOrder);+ * // fields = [+ * // {name: 'time', format: 'YYYY-M-D H:m:s', tableFieldIndex: 1, type: 'timestamp'},+ * // {name: 'value', format: '', tableFieldIndex: 4, type: 'integer'},+ * // {name: 'surge', format: '', tableFieldIndex: 5, type: 'real'},+ * // {name: 'isTrip', format: '', tableFieldIndex: 6, type: 'boolean'},+ * // {name: 'zeroOnes', format: '', tableFieldIndex: 7, type: 'integer'}];*- * @param {array} data- * @param {array} fieldOrder- * @returns {array} formatted fields*/export function getFieldsFromData(data, fieldOrder) {// add a check for epoch timestamp@@ -170,9 +245,6 @@ export function getFieldsFromData(data, fieldOrder) {orderedArray[index] = {name,format,-- // need this for mapbuilder conversion: filter type detection- // category,tableFieldIndex: index + 1,type: analyzerTypeToFieldType(type)};@@ -185,7 +257,7 @@ export function getFieldsFromData(data, fieldOrder) {* pass in an array of field names, rename duplicated one* and return a map from old field index to new name*- * @param {array} fieldOrder+ * @param {Array} fieldOrder* @returns {Object} new field name by index*/export function renameDuplicateFields(fieldOrder) {@@ -213,10 +285,10 @@ export function renameDuplicateFields(fieldOrder) {}/**- * Map Analyzer types to local field types+ * Convert type-analyzer output to kepler.gl field types** @param {string} aType- * @returns {string} corresponding type in ALL_FIELD_TYPES+ * @returns {string} corresponding type in `ALL_FIELD_TYPES`*//* eslint-disable complexity */export function analyzerTypeToFieldType(aType) {@@ -266,8 +338,26 @@ export function analyzerTypeToFieldType(aType) {}/* eslint-enable complexity */-/*- * Process rawData where each row is an object+/**+ * Process data where each row is an object, output can be passed to [`addDataToMap`](../actions/actions.md#adddatatomap)+ * @param {Object} rawData an array of object+ * @returns {Object} dataset containing `fields` and `rows`+ * @public+ * @example+ * import {addDataToMap} from 'kepler.gl/actions';+ * import {processRowObject} from 'kepler.gl/processors';+ *+ * const data = [+ * {lat: 31.27, lng: 127.56, value: 3},+ * {lat: 31.22, lng: 126.26, value: 1}+ * ];+ *+ * dispatch(addDataToMap({+ * datasets: {+ * info: {label: 'My Data', id: 'my_data'},+ * data: processRowObject(data)+ * }+ * }));*/export function processRowObject(rawData) {if (!rawData.length) {@@ -285,8 +375,41 @@ export function processRowObject(rawData) {}/**+ * Process GeoJSON [`FeatureCollection`](http://wiki.geojson.org/GeoJSON_draft_version_6#FeatureCollection),+ * output a data object with `{fields: [], rows: []}`.+ * The data object can be wrapped in a `dataset` and pass to [`addDataToMap`](../actions/actions.md#adddatatomap)+ *+ * @param {Object} rawData raw geojson feature collection+ * @returns {Object} dataset containing `fields` and `rows`+ * @public+ * @example+ * import {addDataToMap} from 'kepler.gl/actions';+ * import {processGeojson} from 'kepler.gl/processors';+ *+ * const geojson = {+ * "type" : "FeatureCollection",+ * "features" : [{+ * "type" : "Feature",+ * "properties" : {+ * "capacity" : "10",+ * "type" : "U-Rack"+ * },+ * "geometry" : {+ * "type" : "Point",+ * "coordinates" : [ -71.073283, 42.417500 ]+ * }+ * }]+ * };*- * @param {Object} rawData - raw geojson feature collection+ * dispatch(addDataToMap({+ * datasets: {+ * info: {+ * label: 'Sample Taxi Trips in New York City',+ * id: 'test_trip_data'+ * },+ * data: processGeojson(geojson)+ * }+ * }));*/export function processGeojson(rawData) {const normalizedGeojson = normalize(rawData);@@ -332,8 +455,9 @@ export function processGeojson(rawData) {/*** On export data to csv- * @param data- * @param fields+ * @param {Array<Array>} data `dataset.allData` or filtered data `dataset.data`+ * @param {Array<Object>} fields `dataset.fields`+ * @returns {string} csv string*/export function formatCsv(data, fields) {const columns = fields.map(f => f.name);@@ -353,6 +477,7 @@ export function formatCsv(data, fields) {}/**+ * Validate input data, adding missing field types, rename duplicate columns* @param data* @returns {{allData: Array, fields: Array}}*/@@ -386,7 +511,7 @@ export function validateInputData(data) {// check if all fields has name, format and typeconst allValid = fields.every((f, i) => {if (typeof f !== 'object') {- assert(`fields needs to be an array of object, but find ${f}`);+ assert(`fields needs to be an array of object, but find ${typeof f}`);return false;}@@ -425,21 +550,21 @@ export function validateInputData(data) {}/**- * Process kepler.gl json to be load by addDataToMap+ * Process saved kepler.gl json to be pass to [`addDataToMap`](../actions/actions.md#adddatatomap).+ * The json object should contain `datasets` and `config`.* @param {Object} rawData+ * @param {Array} rawData.datasets+ * @param {Object} rawData.config+ * @returns {Object} datasets and config `{datasets: {}, config: {}}`+ * @public+ * @example+ * import {addDataToMap} from 'kepler.gl/actions';+ * import {processKeplerglJSON} from 'kepler.gl/processors';+ *+ * dispatch(addDataToMap(processKeplerglJSON(keplerGlJson)));*/export function processKeplerglJSON(rawData) {return rawData? KeplerGlSchema.load(rawData.datasets, rawData.config): null;}--export default {- processGeojson,- processCsvData,- processRowObject,- processKeplerglJSON,- analyzerTypeToFieldType,- getFieldsFromData,- parseCsvDataByFieldType-};diff --git a/src/processors/index.js b/src/processors/index.jsindex 426f9f6c1..dad374a8d 100644--- a/src/processors/index.js+++ b/src/processors/index.js@@ -19,7 +19,15 @@// THE SOFTWARE.// Data Processor-export {default as default} from './data-processor';+export {+ processGeojson,+ processCsvData,+ processRowObject,+ processKeplerglJSON,+ analyzerTypeToFieldType,+ getFieldsFromData,+ parseCsvRowsByFieldType+} from './data-processor';// File Handlersexport {getFileHandler, getFileType} from './file-handler';diff --git a/test/node/utils/data-processor-test.js b/test/node/utils/data-processor-test.jsindex 1e0c3b356..46dc1a41b 100644--- a/test/node/utils/data-processor-test.js+++ b/test/node/utils/data-processor-test.js@@ -25,7 +25,7 @@ import {geojsonData, fields as geojsonFields, rows as geojsonRows} from 'test/fiimport {getFieldsFromData,getSampleForTypeAnalyze,- parseCsvDataByFieldType,+ parseCsvRowsByFieldType,processCsvData,processGeojson} from 'processors/data-processor';@@ -136,7 +136,7 @@ test('Processor => processGeojson', t => {});-test('Processor -> parseCsvDataByFieldType -> real', t => {+test('Processor -> parseCsvRowsByFieldType -> real', t => {const field = {type: ALL_FIELD_TYPES.real@@ -164,12 +164,12 @@ test('Processor -> parseCsvDataByFieldType -> real', t => {[1550.0]];- parseCsvDataByFieldType(rows, field, 0);+ parseCsvRowsByFieldType(rows, field, 0);t.same(rows, expected, 'should parsed reals properly');t.end();});-test('Processor -> parseCsvDataByFieldType -> integer', t => {+test('Processor -> parseCsvRowsByFieldType -> integer', t => {const field = {type: ALL_FIELD_TYPES.integer@@ -193,12 +193,12 @@ test('Processor -> parseCsvDataByFieldType -> integer', t => {[155]];- parseCsvDataByFieldType(rows, field, 0);+ parseCsvRowsByFieldType(rows, field, 0);t.same(rows, expected, 'should parsed ints properly');t.end();});-test('Processor -> parseCsvDataByFieldType -> boolean', t => {+test('Processor -> parseCsvRowsByFieldType -> boolean', t => {const field = {type: ALL_FIELD_TYPES.boolean@@ -227,7 +227,7 @@ test('Processor -> parseCsvDataByFieldType -> boolean', t => {[true]];- parseCsvDataByFieldType(rows, field, 0);+ parseCsvRowsByFieldType(rows, field, 0);t.same(rows, expected, 'should parsed boolean properly');t.end();});