Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[Bug] fix file handler row parsing to support single geojson feature (#1212)
Shan Heff01c4ad8f9a
diff --git a/src/processors/file-handler.js b/src/processors/file-handler.jsindex c851e0efe..5eb565a74 100644--- a/src/processors/file-handler.js+++ b/src/processors/file-handler.js@@ -103,6 +103,7 @@ export async function* readBatch(asyncIterator, fileName) {for await (const batch of asyncIterator) {// Last batch will have this special type and will provide all the root// properties of the parsed document.+ // Only json parse will have `FINAL_RESULT`if (batch.batchType === BATCH_TYPE.FINAL_RESULT) {if (batch.container) {result = {...batch.container};@@ -112,8 +113,9 @@ export async function* readBatch(asyncIterator, fileName) {if (batch.jsonpath && batch.jsonpath.length > 1) {const streamingPath = new _JSONPath(batch.jsonpath);streamingPath.setFieldAtPath(result, batches);- } else {- // The streamed object is a ROW JSON-batch+ } else if (batch.jsonpath && batch.jsonpath.length === 1) {+ // The streamed object is a ROW JSON-batch (jsonpath = '$')+ // row objectsresult = batches;}} else {diff --git a/test/browser/file-handler.js b/test/browser/file-handler.jsindex 41ef13b41..1b092182c 100644--- a/test/browser/file-handler.js+++ b/test/browser/file-handler.js@@ -24,6 +24,10 @@ import {processFileData, readFileInBatches} from 'processors/file-handler';import {csvWithNull} from '../node/processors/file-handler-fixtures';import {dataWithNulls, testFields, parsedDataWithNulls} from 'test/fixtures/test-csv-data';import geojsonString, {+ featureString,+ processedFeature,+ processedFeatureRows,+ processedFeatureFields,processedFields as geojsonFields,processedRows as geojsonRows} from 'test/fixtures/geojson-style';@@ -144,7 +148,7 @@ test('#file-handler -> readFileInBatches.csv -> processFileData', async t => {t.end();});-test('#file-handler -> readFileInBatches.geoJson -> processFileData', async t => {+test('#file-handler -> readFileInBatches.GeoJSON FeatureCollection -> processFileData', async t => {const geojsonFile = new File([geojsonString], 'text-data-1.geojson', {type: ''});const gen = await readFileInBatches({file: geojsonFile, fileCache: []});@@ -266,6 +270,100 @@ test('#file-handler -> readFileInBatches.geoJson -> processFileData', async t =>t.end();});+test('#file-handler -> readFileInBatches.GeoJSON Single Feature -> processFileData', async t => {+ const geojsonFile = new File([featureString], 'text-data-1.geojson', {type: ''});+ const gen = await readFileInBatches({file: geojsonFile, fileCache: []});++ // metadata batch+ const batch1 = await gen.next();++ const expected1 = {+ value: {+ batchType: 'metadata',+ metadata: {_loader: {}, _context: {}},+ data: [],+ bytesUsed: 0,+ progress: {rowCount: 0, rowCountInBatch: 0, percent: 0},+ fileName: 'text-data-1.geojson'+ },+ done: false+ };++ t.deepEqual(+ Object.keys(batch1.value).sort(),+ Object.keys(expected1.value).sort(),+ 'value should have same keyss'+ );++ t.equal(batch1.value.batchType, expected1.value.batchType, 'batch1.batchType should be the same');+ t.equal(batch1.value.fileName, expected1.value.fileName, 'batch1.fileName should be the same');+ t.deepEqual(batch1.value.data, expected1.value.data, 'batch1.data should be the same');+ t.deepEqual(+ batch1.value.progress,+ expected1.value.progress,+ 'batch1.progress should be the same'+ );++ // final result batch+ const batch2 = await gen.next();+ const expected2 = {+ value: {+ batchType: 'final-result',+ container: processedFeature,+ jsonpath: null,+ data: processedFeature,+ schema: null,+ progress: {rowCount: 0, rowCountInBatch: 0},+ fileName: 'text-data-1.geojson'+ },+ done: false+ };++ t.deepEqual(+ batch2.value.batchType,+ expected2.value.batchType,+ 'batch2 batchType should be a final-result'+ );+ t.deepEqual(+ batch2.value.data,+ expected2.value.data,+ 'batch2 data should be a single geojson feature'+ );+ t.deepEqual(+ batch2.value.container,+ expected2.value.container,+ 'batch2 container should be a single geojson feature'+ );+ t.equal(batch2.value.jsonpath, expected2.value.jsonpath, 'batch2 jsonpath should be null');+ t.deepEqual(batch2.value.progress, expected2.value.progress, 'batch2 progress should be correct');++ const batch3 = await gen.next();+ t.deepEqual(batch3, {value: undefined, done: true}, 'batch3 should be done');++ // process geojson data received+ const processed = await processFileData({content: batch2.value, fileCache: []});+ const expectedInfo = {label: 'text-data-1.geojson', format: 'geojson'};++ t.equal(processed.length, 1, 'processFileData should return 1 result');+ t.ok(processed[0].info, 'processFileData should have info');+ t.ok(processed[0].data, 'processFileData should have data');+ t.deepEqual(processed[0].info, expectedInfo, 'info should be correct');+ const {fields, rows} = processed[0].data;++ fields.forEach((f, i) => {+ t.deepEqual(+ f,+ processedFeatureFields[i],+ `should process correct geojson field ${processedFeatureFields[i].name}`+ );+ });+ rows.forEach((r, i) => {+ t.deepEqual(r, processedFeatureRows[i], `should process geojson row ${i} correctly`);+ });++ t.end();+});+test('#file-handler -> readFileInBatches.row -> processFileData', async t => {const fileName = 'row-data.json';const rowFile = new File([rowDataString], fileName, {type: ''});diff --git a/test/fixtures/geojson-style.js b/test/fixtures/geojson-style.jsindex d8a95161a..cf3bf2a10 100644--- a/test/fixtures/geojson-style.js+++ b/test/fixtures/geojson-style.js@@ -21,6 +21,54 @@const geojsonString = `{"type":"FeatureCollection","features":[{"geometry":{"type":"Point","coordinates":[-105.15471672508885,39.98626910199207,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"radius":1,"id":"a1398a11-d1ce-421c-bf66-a456ff525de9"}},{"geometry":{"type":"Point","coordinates":[-105.1549804351595,39.98397605319894,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"94ed2c0f-27ce-416e-b3b4-6c00954b41f0"}},{"geometry":{"type":"Point","coordinates":[-105.15478982047146,39.98296589543148,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"radius":3,"id":"1f59be4c-82e8-4644-b3cf-4c1c0510cbb2"}},{"geometry":{"type":"Point","coordinates":[-105.15449343321012,39.98437157892626,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"radius":5,"id":"de9210a0-c48c-41bf-8463-4b0734d402c0"}},{"geometry":{"type":"Point","coordinates":[-105.15484576666667,39.98312416666666,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"9438eea5-dde0-4e5e-bdf5-7420eedbc419"}},{"geometry":{"type":"Point","coordinates":[-105.15494596666667,39.98422703333333,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"radius":0.5,"id":"9ecefa7c-4fc6-46b7-936c-580a6084ff47"}},{"geometry":{"type":"Point","coordinates":[-105.15522212737501,39.98433057912913,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"696a33f6-3ff2-45a9-8a11-ab541aa2152f"}},{"geometry":{"type":"Point","coordinates":[-105.15447046666667,39.9834028,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"a5b27f57-37a4-4576-ac4e-24dfb57730c3"}},{"geometry":{"type":"Point","coordinates":[-105.15487273333333,39.98379046666667,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"7a640dbe-e0ca-4c50-a43a-b65ec33305b5"}},{"geometry":{"type":"Point","coordinates":[-105.154786,39.986418799999996,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"154cc349-c8b3-4b41-a008-364df1e6d83d"}},{"geometry":{"type":"Point","coordinates":[-105.15456956666667,39.984760566666665,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"932d45a2-bc4b-4825-9b9f-70b2415753b5"}},{"geometry":{"type":"Point","coordinates":[-105.15503543214001,39.983469561355626,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"a178d24c-1ea3-46cd-8762-3669d7b9d722"}},{"geometry":{"type":"Point","coordinates":[-105.15462816666667,39.984541766666666,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"41f8b94e-266c-450a-a7dc-60c159370ddb"}},{"geometry":{"type":"Point","coordinates":[-105.15494077274344,39.98493993797259,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"99ee26ca-13e1-4bef-bcd2-12cfc290d6ae"}},{"geometry":{"type":"Point","coordinates":[-105.15451193333332,39.98379226666666,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"de5103bb-2808-46b4-8012-76ceb5511fa9"}},{"geometry":{"type":"Point","coordinates":[-105.1547839,39.98589206666667,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"c5caef3d-abc9-4044-bb14-2f5999979d8d"}},{"geometry":{"type":"Point","coordinates":[-105.15513263333332,39.98269536666667,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"e0162a54-e2b2-4076-be24-4f550f0ac651"}},{"geometry":{"type":"Point","coordinates":[-105.1546226,39.98516726666667,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"e53ab272-7f6a-49b8-93f1-693bb57aa9f8"}},{"geometry":{"type":"Point","coordinates":[-105.1545185,39.98417893333333,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"fdc3fff3-5c35-430b-8bb4-9517473a8dc1"}},{"geometry":{"type":"Point","coordinates":[-105.15476356666666,39.98555806666667,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"f342e7bd-771d-4bf1-ad57-40d435f54fa0"}},{"geometry":{"type":"Point","coordinates":[-105.15468952992738,39.985742351897976,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"dbda33a4-b194-40e8-88a6-5e8816a97377"}},{"geometry":{"type":"Point","coordinates":[-105.1545091,39.983995666666665,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"5564b2fb-8368-4f81-a8b4-bebb30cbaaed"}},{"geometry":{"type":"Point","coordinates":[-105.15508416666667,39.985163,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"835f956f-6a86-4a7c-b1d6-3f32e4955a31"}},{"geometry":{"type":"Point","coordinates":[-105.15513636666667,39.983126066666664,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"8f8effff-f89d-417c-8e44-c8561f9f1eec"}},{"geometry":{"type":"Point","coordinates":[-105.15478573333333,39.98606776666667,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"59552665-bdd6-4c0c-bd0d-5ee79ea4848b"}},{"geometry":{"type":"Point","coordinates":[-105.1547872,39.9866052,0]},"type":"Feature","properties":{"fillColor":[77,193,156],"id":"32df5401-38d1-4e96-8d1e-a29d2c7c3955"}}]}`;export default geojsonString;+export const featureString = `{"type":"Feature","geometry":{"type":"LineString","coordinates":[[41.8817441,-87.6335398],[41.88237441,-87.6331298]]},"properties":{"lat":41.8817441,"lng":-87.6335398,"icon":"place"}}`;++export const processedFeature = {+ type: 'Feature',+ geometry: {+ type: 'LineString',+ coordinates: [+ [41.8817441, -87.6335398],+ [41.88237441, -87.6331298]+ ]+ },+ properties: {+ lat: 41.8817441,+ lng: -87.6335398,+ icon: 'place'+ }+};++export const processedFeatureFields = [+ {name: '_geojson', format: '', tableFieldIndex: 1, type: 'geojson', analyzerType: 'GEOMETRY'},+ {name: 'lat', format: '', tableFieldIndex: 2, type: 'real', analyzerType: 'FLOAT'},+ {name: 'lng', format: '', tableFieldIndex: 3, type: 'real', analyzerType: 'FLOAT'},+ {name: 'icon', format: '', tableFieldIndex: 4, type: 'string', analyzerType: 'STRING'}+];++export const processedFeatureRows = [+ [+ {+ type: 'Feature',+ geometry: {+ type: 'LineString',+ coordinates: [+ [41.8817441, -87.6335398],+ [41.88237441, -87.6331298]+ ]+ },+ properties: {+ lat: 41.8817441,+ lng: -87.6335398,+ icon: 'place'+ }+ },+ 41.8817441,+ -87.6335398,+ 'place'+ ]+];+export const processedFields = [{name: '_geojson', format: '', tableFieldIndex: 1, type: 'geojson', analyzerType: 'GEOMETRY'},{name: 'fillColor', format: '', tableFieldIndex: 2, type: 'geojson', analyzerType: 'ARRAY'},