Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix(processors): reject unsupported GeoJSON CRS declarations (#3904)
Igor Dykhtad05948d358aa
diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.tsindex eb4040d0d..389b55ab5 100644--- a/src/processors/src/data-processor.ts+++ b/src/processors/src/data-processor.ts@@ -310,6 +310,59 @@ export function processRowObject(rawData: unknown[]): ProcessorResult {return processCsvData(rows, keys);}+const WGS84_CRS_NAMES = [+ /^(?:EPSG:4326|OGC:CRS84h?|CRS84h?|WGS ?84)$/i,+ /^urn:(?:x-)?ogc:def:crs:(?:EPSG:(?:[\d.]*:)?4326|OGC:(?:(?:[\d.]*:)?CRS84h?|2:84))$/i,+ /^https?:\/\/www\.opengis\.net\/def\/crs\/(?:EPSG\/[\d.]+\/4326|OGC\/[\d.]+\/CRS84h?)$/i,+ /^https?:\/\/www\.opengis\.net\/gml\/srs\/epsg\.xml#4326$/i+];++function validateGeojsonCrs(rawData: unknown): void {+ const pending = [rawData];+ while (pending.length) {+ const value = pending.pop();+ if (!isPlainObject(value)) continue;++ const {crs} = value;+ if (isPlainObject(crs) && isPlainObject(crs.properties)) {+ const identifier =+ crs.type === 'name'+ ? crs.properties.name+ : crs.type === 'link'+ ? crs.properties.href+ : undefined;+ // Preserve existing handling of absent, null, and malformed declarations.+ // Recognize known identifiers without fetching linked CRS definitions.+ if (+ typeof identifier === 'string' &&+ identifier.trim() &&+ !WGS84_CRS_NAMES.some(pattern => pattern.test(identifier.trim()))+ ) {+ throw new Error(+ 'Read File Failed: GeoJSON uses an unsupported coordinate reference system. ' ++ 'Convert coordinates to WGS84 (EPSG:4326) before importing.'+ );+ }+ }++ // CRS can be declared on collections, features, or their geometries, but a+ // feature property named "crs" is ordinary data and must not be inspected.+ if (value.type === 'Feature') {+ pending.push(value.geometry);+ } else {+ const children =+ value.type === 'FeatureCollection'+ ? value.features+ : value.type === 'GeometryCollection'+ ? value.geometries+ : null;+ if (Array.isArray(children)) {+ for (const child of children) pending.push(child);+ }+ }+ }+}+/*** Process GeoJSON [`FeatureCollection`](http://wiki.geojson.org/GeoJSON_draft_version_6#FeatureCollection),* output a data object with `{fields: [], rows: []}`.@@ -349,6 +402,7 @@ export function processRowObject(rawData: unknown[]): ProcessorResult {* }));*/export function processGeojson(rawData: unknown): ProcessorResult {+ validateGeojsonCrs(rawData);const normalizedGeojson = normalize(rawData);if (!normalizedGeojson || !Array.isArray(normalizedGeojson.features)) {diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex ebe2be215..ce3917075 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -32,6 +32,8 @@ import {processCsvData,processGeojson,processArrowBatches,+ processFileData,+ readFileInBatches,loadExternallyHostedDataset,remoteDatasetFromUrl} from '@kepler.gl/processors';@@ -8106,6 +8108,74 @@ test('#visStateReducer -> TOGGLE_TABLE_COLUMN_HIDDEN and SHOW_ALL_TABLE_COLUMNS't.end();});+test('#visStateReducer -> unsupported streamed GeoJSON CRS never commits partial data', async t => {+ drainTasksForTesting();+ const previousProgressiveLoading = getApplicationConfig().useArrowProgressiveLoading;+ initApplicationConfig({useArrowProgressiveLoading: true});+ try {+ const initialState = CloneDeep(InitialState).visState;+ const feature = {+ type: 'Feature',+ properties: {},+ geometry: {type: 'Point', coordinates: [1000000, 1000000]}+ };+ // Put CRS after features so streaming cannot discover it until the final container.+ const file = new File(+ [+ JSON.stringify({+ type: 'FeatureCollection',+ features: [feature],+ crs: {type: 'name', properties: {name: 'EPSG:3857'}}+ })+ ],+ 'projected.geojson',+ {type: 'application/geo+json'}+ );+ const nextFile = new File(['lat,lng\n20,10'], 'valid.csv', {type: 'text/csv'});+ let state = reducer(initialState, VisStateActions.loadFiles([file, nextFile]));+ const [loadTask] = drainTasksForTesting();+ const iterator = await readFileInBatches({file, loadOptions: {worker: false}});+ state = reducer(state, succeedTaskInTest(loadTask, iterator));+ let [task] = drainTasksForTesting();+ let streamedRows = false;+ while (task.type === 'UNWRAP') {+ const batch = await task.payload;+ streamedRows ||= Array.isArray(batch.value?.data) && batch.value.data.length > 0;+ state = reducer(state, succeedTaskInTest(task, batch));+ t.equal(state.datasets, initialState.datasets, 'streamed GeoJSON never adds partial data');+ t.deepEqual(state.fileLoading.fileCache, [], 'streamed GeoJSON never enters the file cache');+ const tasks = drainTasksForTesting();+ t.equal(tasks.length, 1, 'GeoJSON should not schedule a progressive dataset commit');+ [task] = tasks;+ }+ t.ok(streamedRows, 'exercise feature batches before the final CRS declaration');+ t.equal(task.type, 'PROCESS_FILE_CONTENT', 'process the completed GeoJSON file');+ try {+ await processFileData(task.payload);+ t.fail('the completed file must reject its unsupported CRS');+ } catch (error) {+ t.match(error.message, /WGS84.*EPSG:4326/, 'surface an actionable import error');+ const warn = sinon.stub(Console, 'warn');+ try {+ state = reducer(state, errorTaskInTest(task, error));+ } finally {+ warn.restore();+ }+ t.equal(state.fileLoadingProgress[file.name].error, error, 'record the file error');+ t.equal(state.datasets, initialState.datasets, 'failed import preserves existing datasets');+ t.deepEqual(state.fileLoading.fileCache, [], 'failed import does not leave a partial cache');+ const [delay] = drainTasksForTesting();+ state = reducer(state, succeedTaskInTest(delay));+ const [nextLoad] = drainTasksForTesting();+ t.equal(nextLoad.payload.file, nextFile, 'continue loading the next file after the error');+ }+ } finally {+ initApplicationConfig({useArrowProgressiveLoading: previousProgressiveLoading});+ drainTasksForTesting();+ }+ t.end();+});+test('#visStateReducer -> LOAD_FILES', async t => {drainTasksForTesting();diff --git a/test/node/utils/data-processor-test.js b/test/node/utils/data-processor-test.jsindex ee39e56cd..c7d0e95aa 100644--- a/test/node/utils/data-processor-test.js+++ b/test/node/utils/data-processor-test.js@@ -855,6 +855,96 @@ test('Processor => processGeojson:invalid geojson', t => {t.end();});+test('Processor => processGeojson: rejects explicit unsupported CRS declarations', t => {+ for (const crs of [+ {type: 'name', properties: {name: 'EPSG:3857'}},+ {type: 'name', properties: {name: 'urn:ogc:def:crs:EPSG::2263'}},+ {type: 'name', properties: {name: 'urn:ogc:def:crs:OGC:2:27'}},+ {type: 'name', properties: {name: 'unknown-projection'}},+ {type: 'link', properties: {href: 'https://example.com/projected-crs'}}+ ]) {+ const geometry = {type: 'Point', coordinates: [1000000, 1000000]};+ const feature = {type: 'Feature', properties: {}, geometry};+ for (const input of [+ {type: 'FeatureCollection', crs, features: [feature]},+ {...feature, crs},+ {...feature, geometry: {...geometry, crs}},+ {type: 'FeatureCollection', features: [{...feature, crs}]},+ {type: 'GeometryCollection', geometries: [{...geometry, crs}]}+ ]) {+ t.throws(+ () => processGeojson(input),+ /WGS84.*EPSG:4326/,+ 'unsupported CRS should explain the required coordinate system'+ );+ }+ }+ t.end();+});++test('Processor => processGeojson: preserves supported and undeclared CRS data', t => {+ const aliases = [+ 'EPSG:4326',+ 'epsg:4326',+ 'WGS84',+ 'WGS 84',+ 'CRS84',+ 'OGC:CRS84',+ 'CRS84h',+ 'OGC:CRS84h',+ 'urn:ogc:def:crs:EPSG::4326',+ 'urn:ogc:def:crs:EPSG:6.6:4326',+ 'urn:x-ogc:def:crs:EPSG:4326',+ 'urn:ogc:def:crs:OGC:1.3:CRS84',+ 'urn:ogc:def:crs:OGC:2:84',+ 'urn:x-ogc:def:crs:OGC:2:84',+ 'urn:ogc:def:crs:OGC:0:CRS84h',+ 'http://www.opengis.net/def/crs/EPSG/0/4326',+ 'https://www.opengis.net/def/crs/OGC/1.3/CRS84',+ 'http://www.opengis.net/def/crs/OGC/0/CRS84h',+ 'https://www.opengis.net/def/crs/OGC/0/CRS84h',+ 'http://www.opengis.net/gml/srs/epsg.xml#4326'+ ];+ const features = [+ {type: 'Feature', properties: {crs: 'an ordinary property'}, geometry: null},+ {+ type: 'Feature',+ properties: {crs: {type: 'name', properties: {name: 'EPSG:3857'}}},+ geometry: {type: 'Point', coordinates: [181, 20, 100]}+ },+ {type: 'Feature', properties: {}, geometry: {type: 'Polygon', coordinates: []}}+ ];+ const expected = processGeojson({type: 'FeatureCollection', features: cloneDeep(features)});+ for (const crs of [+ undefined,+ null,+ ...aliases.map(name => ({type: 'name', properties: {name}})),+ {type: 'link', properties: {href: 'http://www.opengis.net/def/crs/EPSG/0/4326'}},+ {type: 'link', properties: {href: 'https://www.opengis.net/def/crs/OGC/1.3/CRS84'}},+ {type: 'link', properties: {href: 'http://www.opengis.net/def/crs/OGC/0/CRS84h'}},+ {type: 'link', properties: {href: 'https://www.opengis.net/def/crs/OGC/0/CRS84h'}}+ ]) {+ const data = processGeojson({type: 'FeatureCollection', crs, features: cloneDeep(features)});+ t.deepEqual(data.rows, expected.rows, 'CRS recognition must not rewrite or drop geometry');+ t.deepEqual(+ processGeojson({type: 'FeatureCollection', crs, features: []}).rows,+ [],+ 'empty collections remain valid'+ );+ }+ t.end();+});++test('Processor => processGeojson: leaves malformed CRS schema validation unchanged', t => {+ for (const crs of [{}, 'EPSG:3857', {type: 'name'}, {type: 'name', properties: {name: 3857}}]) {+ t.doesNotThrow(+ () => processGeojson({type: 'FeatureCollection', crs, features: []}),+ 'malformed CRS declarations retain their existing behavior'+ );+ }+ t.end();+});+test('Processor => processGeojson: parse rows', t => {const testGeoData = {type: 'FeatureCollection',