Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[Bug] handle header-only remote CSV snapshots (#3891)
Igor Dykhta44fc426fa606
diff --git a/src/processors/src/remote-file.ts b/src/processors/src/remote-file.tsindex 1aebf23a4..e2b59874d 100644--- a/src/processors/src/remote-file.ts+++ b/src/processors/src/remote-file.ts@@ -20,6 +20,7 @@ import {readFileInBatches} from './file-handler';import {FileCacheItem} from './types';+import {KeplerCSVLoader} from './kepler-csv-loader';export type KeplerRemoteFile = File & {keplerSourceUrl?: string;@@ -353,6 +354,50 @@ function isTabularProcessorResult(data: unknown): data is NonNullable<ProcessorR);}+async function matchesCsvHeader(text: string, fields: string[]): Promise<boolean> {+ const escape = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');+ const blankLines = '(?:[ \\t]*[\\r\\n])*';+ // Keep exact quote validation: the CSV parser accepts some malformed records.+ const matchesRecord = (header: string[], strict: boolean) =>+ [',', '\t', ';', '|'].some(delimiter => {+ const record = header+ .map(name => {+ const quoted = escape(`"${name.replace(/"/g, '""')}"`);+ const needsQuotes =+ /["\r\n]/.test(name) || (strict ? /[,\t;|]/.test(name) : name.includes(delimiter));+ return needsQuotes ? quoted : `(?:${escape(name)}|${quoted})`;+ })+ .join(escape(delimiter));+ return new RegExp(`^\\uFEFF?${blankLines}${record}${blankLines}$`).test(text);+ });+ // Quoted names remain unambiguous even when the loader guesses another delimiter.+ if (fields.some(name => name.trim()) && matchesRecord(fields, true)) return true;+ // A lone bare name containing delimiters has no separator to disambiguate it.+ if (fields.length === 1) return false;++ // Reuse the same streaming parser as import, exposing the header as a row.+ const rows: string[][] = [];+ for await (const batch of KeplerCSVLoader.parseInBatches!([new TextEncoder().encode(text)], {+ csv: {+ shape: 'array-row-table',+ // @ts-expect-error loaders.gl accepts false but its type only declares 'auto'.+ header: false+ }+ })) {+ rows.push(...batch.data);+ }+ if (rows.length !== 1) return false;+ const header = rows[0];+ // processRowObject uses Object.keys, which sorts integer-named columns.+ const names = Object.keys(Object.fromEntries(header.map(name => [name, null])));+ return (+ header.length === fields.length &&+ names.length === fields.length &&+ names.every((name, i) => name === fields[i]) &&+ matchesRecord(header, false)+ );+}+/*** Fetch, parse, and process a remote file into Kepler dataset data.* Used when reloading an `externally-hosted` dataset from a saved map config@@ -366,6 +411,8 @@ export async function loadExternallyHostedDataset(metadata: {lastModified?: string;bypassCache?: boolean;onProgress?: (progress: RemoteLoadProgress) => void;+ /** Established CSV columns, supplied only when refreshing an existing table. */+ csvHeader?: string[];}): Promise<LoadExternallyHostedDatasetResult> {const {source, format, etag, lastModified, size, bypassCache, onProgress} = metadata;const fetched = await fetchRemoteFile(source, {@@ -405,6 +452,24 @@ export async function loadExternallyHostedDataset(metadata: {for await (const batch of batches) {content = batch as ProcessFileDataContent;}+ // CSVLoader emits no data batch for a header-only file. Keep its populated+ // parsing path intact; recognize only a header matching this existing table.+ if (+ metadata.csvHeader?.length &&+ KeplerCSVLoader.extensions.some(extension =>+ fetched.file!.name.toLowerCase().endsWith(`.${extension}`)+ ) &&+ (!content || (Array.isArray(content.data) && !content.data.length)) &&+ (await matchesCsvHeader(await fetched.file.text(), metadata.csvHeader))+ ) {+ return {+ data: {fields: [], rows: []},+ notModified: false,+ etag: fetched.etag,+ lastModified: fetched.lastModified,+ size: fetched.size+ };+ }if (!content) {throw new Error(`No data loaded from ${source}`);}diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.tsindex b8c64bdd1..18ab26b3e 100644--- a/src/reducers/src/vis-state-updaters.ts+++ b/src/reducers/src/vis-state-updaters.ts@@ -3764,6 +3764,7 @@ async function refreshExternallyHostedTable({size?: number;etag?: string;lastModified?: string;+ csvHeader?: string[];};onProgress: (progress: {loaded: number; total?: number; percent: number}) => void;}) {@@ -3817,6 +3818,7 @@ export function refreshDatasetUpdater(arg: {source: nextDataset.metadata.source as string,format: getRemoteSourceFormat(nextDataset.metadata),+ csvHeader: nextDataset.fields.map(field => field.name),size: typeof nextDataset.metadata.size === 'number' ? nextDataset.metadata.size : undefined,etag: typeof nextDataset.metadata.etag === 'string' ? nextDataset.metadata.etag : undefined,lastModified:diff --git a/test/node/processors/remote-file-test.js b/test/node/processors/remote-file-test.jsindex 1cb4879cd..cb9b57d3b 100644--- a/test/node/processors/remote-file-test.js+++ b/test/node/processors/remote-file-test.js@@ -8,9 +8,150 @@ import {getExtensionFromUrl,getFileNameForRemoteUrl,getMimeTypeForFormat,- isRemoteDatasetUrl+ isRemoteDatasetUrl,+ loadExternallyHostedDataset} from '@kepler.gl/processors';+test('#remote-file -> recognizes only the established CSV header', async t => {+ const originalFetch = global.fetch;+ const metadata = {+ source: 'https://example.com/live.csv',+ csvHeader: ['lat', 'lng', 'value']+ };+ try {+ for (const body of [+ 'lat,lng,value',+ '\ufeff"lat",lng,"value"\r\n',+ '\nlat,lng,value\n\n',+ 'lat\tlng\tvalue\n',+ 'lat;lng;value\n',+ 'lat|lng|value\n'+ ]) {+ global.fetch = async () => new Response(body, {headers: {etag: 'empty'}});+ const result = await loadExternallyHostedDataset(metadata);+ t.deepEqual(result.data, {fields: [], rows: []}, 'clears rows using established types');+ t.equal(result.notModified, false, 'empty is a new snapshot');+ t.equal(result.etag, 'empty', 'retains response validators');+ }+ const csvHeader = ['la,t', 'ln\ng', 'say "hi"', 'a.*'];+ global.fetch = async () => new Response('"la,t","ln\ng","say ""hi""",a.*\n');+ const result = await loadExternallyHostedDataset({...metadata, csvHeader});+ t.deepEqual(result.data.rows, [], 'recognizes quoted delimiters, newlines and escaped quotes');++ for (const body of [+ '',+ '\n \t\r\n',+ '"lat,lng,value\n',+ 'lat,lng,"value',+ '"lat"oops,lng,value\n',+ 'lat"oops,lng,value\n',+ 'value,lng,lat\n',+ 'lat,lng\n',+ 'lat,lng,value,new\n'+ ]) {+ global.fetch = async () => new Response(body);+ try {+ await loadExternallyHostedDataset(metadata);+ t.fail('blank, malformed or changed headers must not clear the table');+ } catch (error) {+ t.ok(error instanceof Error, 'rejects unsupported empty snapshots');+ }+ }+ global.fetch = async () => new Response('lat,lng,value\n');+ try {+ await loadExternallyHostedDataset({source: metadata.source});+ t.fail('initially empty imports must retain their existing error behavior');+ } catch (error) {+ t.ok(error instanceof Error, 'requires an established table');+ }+ } finally {+ global.fetch = originalFetch;+ }+ t.end();+});++test('#remote-file -> header matching cannot reinterpret a column delimiter', async t => {+ const originalFetch = global.fetch;+ const source = 'https://example.com/live.csv';+ const fixtures = [+ ...[',', '\t', ';', '|'].map(delimiter => [+ `"a${delimiter}b"\nvalue\n`,+ `a${delimiter}b\n`,+ `"a${delimiter}b"\n`+ ]),+ [+ 'lat,lng,"a,b,c,d"\n37,-122,1\n38,-123,2\n',+ 'lat;lng;a,b,c,d\n',+ 'lat;lng;"a,b,c,d"\n'+ ]+ ];+ try {+ for (const [populated, ambiguous, quoted] of fixtures) {+ let body = populated;+ global.fetch = async () => new Response(body);+ const loaded = await loadExternallyHostedDataset({source});+ const csvHeader = loaded.data.fields.map(field => field.name);+ body = ambiguous;+ try {+ await loadExternallyHostedDataset({source, csvHeader});+ t.fail('a different delimiter interpretation must not clear existing data');+ } catch (error) {+ t.ok(error instanceof Error, 'rejects the ambiguous header');+ }+ body = quoted;+ const result = await loadExternallyHostedDataset({source, csvHeader});+ t.deepEqual(result.data, {fields: [], rows: []}, 'accepts the quoted established header');+ }+ } finally {+ global.fetch = originalFetch;+ }+ t.end();+});++test('#remote-file -> header-only refresh uses CSV parsing and imported column order', async t => {+ const originalFetch = global.fetch;+ const fixtures = [+ ['live.csv', 'text/csv', 'lat,lng,a|b', '37,-122,1', ['lat', 'lng', 'a|b']],+ ['live.csv', 'text/csv', 'lat;lng;a,b', '37;-122;1', ['lat', 'lng', 'a,b']],+ [+ 'live.tsv',+ 'text/tab-separated-values',+ 'lat\tlng\tvalue',+ '37\t-122\t1',+ ['lat', 'lng', 'value']+ ],+ ['live.dsv', 'text/dsv', 'lat;lng;value', '37;-122;1', ['lat', 'lng', 'value']],+ [+ 'live',+ 'text/tab-separated-values',+ 'lat\tlng\tvalue',+ '37\t-122\t1',+ ['lat', 'lng', 'value']+ ],+ ['live.csv', 'text/csv', '2025,2024,value', '1,2,3', ['2024', '2025', 'value']]+ ];+ try {+ for (const [name, mimeType, header, row, expectedNames] of fixtures) {+ let body = `${header}\n${row}\n`;+ global.fetch = async () => new Response(body, {headers: {'content-type': mimeType}});+ const source = `https://example.com/${name}`;+ const loaded = await loadExternallyHostedDataset({source});+ const csvHeader = loaded.data.fields.map(field => field.name);+ t.deepEqual(csvHeader, expectedNames, 'establishes columns using the normal import path');+ body = `${header}\n`;+ try {+ const result = await loadExternallyHostedDataset({source, csvHeader});+ t.deepEqual(result.data, {fields: [], rows: []}, 'recognizes the unchanged empty snapshot');+ } catch (error) {+ t.fail(`unchanged header-only ${name} rejected: ${error.message}`);+ }+ }+ } finally {+ global.fetch = originalFetch;+ }+ t.end();+});+test('#remote-file -> getExtensionFromUrl', t => {t.equal(getExtensionFromUrl('https://example.com/data.geojson'), 'geojson');t.equal(diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex affc64602..540c63471 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -31,6 +31,7 @@ import {processCsvData,processGeojson,processArrowBatches,+ loadExternallyHostedDataset,remoteDatasetFromUrl} from '@kepler.gl/processors';import {@@ -10154,6 +10155,117 @@ test('VisStateUpdater -> refreshDataset', async t => {t.end();});+test('VisStateUpdater -> CSV refresh preserves layers and filters through 100, 1, 0 and 100 rows', async t => {+ const originalFetch = global.fetch;+ const source = 'https://example.com/live.csv';+ const populated = `lat,lng,value\n${Array.from(+ {length: 100},+ (_, i) => `${37 + i / 1000},${-122 - i / 1000},${i}`+ ).join('\n')}\n`;+ let body = populated;+ global.fetch = async () => new Response(body, {headers: {'content-type': 'text/csv'}});+ const refresh = async state => {+ drainTasksForTesting();+ state = reducer(state, VisStateActions.refreshDataset('remote-csv'));+ const [task] = drainTasksForTesting();+ const action = await new Promise(resolve => {+ task.run((effect, succeed, fail, ctx) => effect(succeed, fail, ctx), resolve, resolve, {+ onProgress: progress => {+ state = reducer(state, progress);+ }+ });+ });+ return reducer(state, action);+ };+ try {+ for (const gpu of [false, true]) {+ const inverted = !gpu;+ body = populated;+ const loaded = await loadExternallyHostedDataset({source, format: 'csv'});+ const datasets = await createNewDataEntryMock({+ info: {id: 'remote-csv', type: DatasetType.EXTERNALLY_HOSTED},+ data: loaded.data,+ metadata: {source, sourceFormat: 'csv'}+ });+ const {props} = PointLayer.findDefaultLayerProps(datasets['remote-csv']);+ const point = new PointLayer({id: 'csv-point', dataId: 'remote-csv', ...props[0]});+ point.updateLayerVisConfig({opacity: 0.37, radius: 9});+ let state = {...INITIAL_VIS_STATE, datasets, layers: [point], layerData: [{}]};+ const initial = calculateLayerData(point, state);+ state = {...state, layers: [initial.layer], layerData: [initial.layerData]};+ const style = {...state.layers[0].config.visConfig};+ const types = datasets['remote-csv'].fields.map(field => field.type);+ state = reducer(state, VisStateActions.addFilter('remote-csv'));+ state = reducer(state, VisStateActions.setFilter(0, 'name', 'value'));+ state = reducer(state, VisStateActions.setFilter(0, 'value', [20, 80]));+ state = reducer(state, VisStateActions.setFilter(0, 'inverted', inverted));+ state = reducer(state, VisStateActions.setFilter(0, 'gpu', gpu));+ const filterId = state.filters[0].id;++ for (const [text, count] of [+ [`lat,lng,value\n40.5,-125.5,${gpu ? 50 : 10}\n`, 1],+ ['lat,lng,value\n', 0],+ [populated, 100]+ ]) {+ body = text;+ state = await refresh(state);+ const ds = state.datasets['remote-csv'];+ t.equal(ds.metadata.refreshStatus, 'idle', 'refresh succeeds');+ t.equal(ds.dataContainer.numRows(), count, 'replaces all rows');+ t.deepEqual(+ ds.fields.map(field => field.type),+ types,+ 'preserves established types'+ );+ t.equal(state.layers.length, 1, 'keeps the configured layer');+ t.equal(state.layers[0].id, 'csv-point', 'keeps layer identity');+ t.equal(state.layers[0].isValid, true, 'keeps the layer valid');+ t.deepEqual(state.layers[0].config.visConfig, style, 'keeps user styling');+ t.equal(state.filters[0].id, filterId, 'keeps filter identity');+ t.deepEqual(state.filters[0].value, [20, 80], 'keeps selected range');+ t.equal(state.filters[0].inverted, inverted, 'keeps Filter out');+ t.equal(state.filters[0].gpu, gpu, 'keeps evaluation mode');+ ds.filterTableCPU(state.filters, state.layers);+ const expected = ds.allIndexes.filter(i => {+ const value = ds.dataContainer.valueAt(i, 2);+ return inverted ? value < 20 || value > 80 : value >= 20 && value <= 80;+ });+ t.deepEqual(ds.filteredIdxCPU, expected, 'export filtering uses the new snapshot');+ t.equal(+ state.layerData[0].data.length,+ gpu ? count : expected.length,+ 'rebuilds layer data'+ );+ if (count === 1) {+ t.deepEqual(+ state.layerData[0].data[0].position.slice(0, 2),+ [-125.5, 40.5],+ 'updates coordinates'+ );+ }+ }+ for (const invalid of ['', 'lat,lng,"value', 'value,lng,lat\n']) {+ body = invalid;+ const previous = state.datasets['remote-csv'].dataContainer;+ const layerData = state.layerData[0];+ state = await refresh(state);+ t.equal(+ state.datasets['remote-csv'].metadata.refreshStatus,+ 'error',+ 'unsupported refresh fails'+ );+ t.equal(state.datasets['remote-csv'].dataContainer, previous, 'failure retains the rows');+ t.equal(state.layerData[0], layerData, 'failure retains rendered data');+ t.deepEqual(state.filters[0].value, [20, 80], 'failure retains the selection');+ }+ }+ } finally {+ global.fetch = originalFetch;+ drainTasksForTesting();+ }+ t.end();+});+test('VisStateUpdater -> refreshDataset keeps Arrow point layers valid through empty snapshots', async t => {const snapshot = values => {const table = arrow.tableFromArrays({