Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[Bug] handle header-only remote CSV snapshots (#3891)

Igor Dykhta44fc426fa606
diff --git a/src/processors/src/remote-file.ts b/src/processors/src/remote-file.ts
index 1aebf23a4..e2b59874d 100644
--- a/src/processors/src/remote-file.ts
+++ b/src/processors/src/remote-file.ts
@@ -20,6 +20,7 @@ import {
readFileInBatches
} from './file-handler';
import {FileCacheItem} from './types';
+import {KeplerCSVLoader} from './kepler-csv-loader';
export type KeplerRemoteFile = File & {
keplerSourceUrl?: string;
@@ -353,6 +354,50 @@ function isTabularProcessorResult(data: unknown): data is NonNullable<ProcessorR
);
}
+async function matchesCsvHeader(text: string, fields: string[]): Promise<boolean> {
+ const escape = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+ const blankLines = '(?:[ \\t]*[\\r\\n])*';
+ // Keep exact quote validation: the CSV parser accepts some malformed records.
+ const matchesRecord = (header: string[], strict: boolean) =>
+ [',', '\t', ';', '|'].some(delimiter => {
+ const record = header
+ .map(name => {
+ const quoted = escape(`"${name.replace(/"/g, '""')}"`);
+ const needsQuotes =
+ /["\r\n]/.test(name) || (strict ? /[,\t;|]/.test(name) : name.includes(delimiter));
+ return needsQuotes ? quoted : `(?:${escape(name)}|${quoted})`;
+ })
+ .join(escape(delimiter));
+ return new RegExp(`^\\uFEFF?${blankLines}${record}${blankLines}$`).test(text);
+ });
+ // Quoted names remain unambiguous even when the loader guesses another delimiter.
+ if (fields.some(name => name.trim()) && matchesRecord(fields, true)) return true;
+ // A lone bare name containing delimiters has no separator to disambiguate it.
+ if (fields.length === 1) return false;
+
+ // Reuse the same streaming parser as import, exposing the header as a row.
+ const rows: string[][] = [];
+ for await (const batch of KeplerCSVLoader.parseInBatches!([new TextEncoder().encode(text)], {
+ csv: {
+ shape: 'array-row-table',
+ // @ts-expect-error loaders.gl accepts false but its type only declares 'auto'.
+ header: false
+ }
+ })) {
+ rows.push(...batch.data);
+ }
+ if (rows.length !== 1) return false;
+ const header = rows[0];
+ // processRowObject uses Object.keys, which sorts integer-named columns.
+ const names = Object.keys(Object.fromEntries(header.map(name => [name, null])));
+ return (
+ header.length === fields.length &&
+ names.length === fields.length &&
+ names.every((name, i) => name === fields[i]) &&
+ matchesRecord(header, false)
+ );
+}
+
/**
* Fetch, parse, and process a remote file into Kepler dataset data.
* Used when reloading an `externally-hosted` dataset from a saved map config
@@ -366,6 +411,8 @@ export async function loadExternallyHostedDataset(metadata: {
lastModified?: string;
bypassCache?: boolean;
onProgress?: (progress: RemoteLoadProgress) => void;
+ /** Established CSV columns, supplied only when refreshing an existing table. */
+ csvHeader?: string[];
}): Promise<LoadExternallyHostedDatasetResult> {
const {source, format, etag, lastModified, size, bypassCache, onProgress} = metadata;
const fetched = await fetchRemoteFile(source, {
@@ -405,6 +452,24 @@ export async function loadExternallyHostedDataset(metadata: {
for await (const batch of batches) {
content = batch as ProcessFileDataContent;
}
+ // CSVLoader emits no data batch for a header-only file. Keep its populated
+ // parsing path intact; recognize only a header matching this existing table.
+ if (
+ metadata.csvHeader?.length &&
+ KeplerCSVLoader.extensions.some(extension =>
+ fetched.file!.name.toLowerCase().endsWith(`.${extension}`)
+ ) &&
+ (!content || (Array.isArray(content.data) && !content.data.length)) &&
+ (await matchesCsvHeader(await fetched.file.text(), metadata.csvHeader))
+ ) {
+ return {
+ data: {fields: [], rows: []},
+ notModified: false,
+ etag: fetched.etag,
+ lastModified: fetched.lastModified,
+ size: fetched.size
+ };
+ }
if (!content) {
throw new Error(`No data loaded from ${source}`);
}
diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts
index b8c64bdd1..18ab26b3e 100644
--- a/src/reducers/src/vis-state-updaters.ts
+++ b/src/reducers/src/vis-state-updaters.ts
@@ -3764,6 +3764,7 @@ async function refreshExternallyHostedTable({
size?: number;
etag?: string;
lastModified?: string;
+ csvHeader?: string[];
};
onProgress: (progress: {loaded: number; total?: number; percent: number}) => void;
}) {
@@ -3817,6 +3818,7 @@ export function refreshDatasetUpdater(
arg: {
source: nextDataset.metadata.source as string,
format: getRemoteSourceFormat(nextDataset.metadata),
+ csvHeader: nextDataset.fields.map(field => field.name),
size: typeof nextDataset.metadata.size === 'number' ? nextDataset.metadata.size : undefined,
etag: typeof nextDataset.metadata.etag === 'string' ? nextDataset.metadata.etag : undefined,
lastModified:
diff --git a/test/node/processors/remote-file-test.js b/test/node/processors/remote-file-test.js
index 1cb4879cd..cb9b57d3b 100644
--- a/test/node/processors/remote-file-test.js
+++ b/test/node/processors/remote-file-test.js
@@ -8,9 +8,150 @@ import {
getExtensionFromUrl,
getFileNameForRemoteUrl,
getMimeTypeForFormat,
- isRemoteDatasetUrl
+ isRemoteDatasetUrl,
+ loadExternallyHostedDataset
} from '@kepler.gl/processors';
+test('#remote-file -> recognizes only the established CSV header', async t => {
+ const originalFetch = global.fetch;
+ const metadata = {
+ source: 'https://example.com/live.csv',
+ csvHeader: ['lat', 'lng', 'value']
+ };
+ try {
+ for (const body of [
+ 'lat,lng,value',
+ '\ufeff"lat",lng,"value"\r\n',
+ '\nlat,lng,value\n\n',
+ 'lat\tlng\tvalue\n',
+ 'lat;lng;value\n',
+ 'lat|lng|value\n'
+ ]) {
+ global.fetch = async () => new Response(body, {headers: {etag: 'empty'}});
+ const result = await loadExternallyHostedDataset(metadata);
+ t.deepEqual(result.data, {fields: [], rows: []}, 'clears rows using established types');
+ t.equal(result.notModified, false, 'empty is a new snapshot');
+ t.equal(result.etag, 'empty', 'retains response validators');
+ }
+ const csvHeader = ['la,t', 'ln\ng', 'say "hi"', 'a.*'];
+ global.fetch = async () => new Response('"la,t","ln\ng","say ""hi""",a.*\n');
+ const result = await loadExternallyHostedDataset({...metadata, csvHeader});
+ t.deepEqual(result.data.rows, [], 'recognizes quoted delimiters, newlines and escaped quotes');
+
+ for (const body of [
+ '',
+ '\n \t\r\n',
+ '"lat,lng,value\n',
+ 'lat,lng,"value',
+ '"lat"oops,lng,value\n',
+ 'lat"oops,lng,value\n',
+ 'value,lng,lat\n',
+ 'lat,lng\n',
+ 'lat,lng,value,new\n'
+ ]) {
+ global.fetch = async () => new Response(body);
+ try {
+ await loadExternallyHostedDataset(metadata);
+ t.fail('blank, malformed or changed headers must not clear the table');
+ } catch (error) {
+ t.ok(error instanceof Error, 'rejects unsupported empty snapshots');
+ }
+ }
+ global.fetch = async () => new Response('lat,lng,value\n');
+ try {
+ await loadExternallyHostedDataset({source: metadata.source});
+ t.fail('initially empty imports must retain their existing error behavior');
+ } catch (error) {
+ t.ok(error instanceof Error, 'requires an established table');
+ }
+ } finally {
+ global.fetch = originalFetch;
+ }
+ t.end();
+});
+
+test('#remote-file -> header matching cannot reinterpret a column delimiter', async t => {
+ const originalFetch = global.fetch;
+ const source = 'https://example.com/live.csv';
+ const fixtures = [
+ ...[',', '\t', ';', '|'].map(delimiter => [
+ `"a${delimiter}b"\nvalue\n`,
+ `a${delimiter}b\n`,
+ `"a${delimiter}b"\n`
+ ]),
+ [
+ 'lat,lng,"a,b,c,d"\n37,-122,1\n38,-123,2\n',
+ 'lat;lng;a,b,c,d\n',
+ 'lat;lng;"a,b,c,d"\n'
+ ]
+ ];
+ try {
+ for (const [populated, ambiguous, quoted] of fixtures) {
+ let body = populated;
+ global.fetch = async () => new Response(body);
+ const loaded = await loadExternallyHostedDataset({source});
+ const csvHeader = loaded.data.fields.map(field => field.name);
+ body = ambiguous;
+ try {
+ await loadExternallyHostedDataset({source, csvHeader});
+ t.fail('a different delimiter interpretation must not clear existing data');
+ } catch (error) {
+ t.ok(error instanceof Error, 'rejects the ambiguous header');
+ }
+ body = quoted;
+ const result = await loadExternallyHostedDataset({source, csvHeader});
+ t.deepEqual(result.data, {fields: [], rows: []}, 'accepts the quoted established header');
+ }
+ } finally {
+ global.fetch = originalFetch;
+ }
+ t.end();
+});
+
+test('#remote-file -> header-only refresh uses CSV parsing and imported column order', async t => {
+ const originalFetch = global.fetch;
+ const fixtures = [
+ ['live.csv', 'text/csv', 'lat,lng,a|b', '37,-122,1', ['lat', 'lng', 'a|b']],
+ ['live.csv', 'text/csv', 'lat;lng;a,b', '37;-122;1', ['lat', 'lng', 'a,b']],
+ [
+ 'live.tsv',
+ 'text/tab-separated-values',
+ 'lat\tlng\tvalue',
+ '37\t-122\t1',
+ ['lat', 'lng', 'value']
+ ],
+ ['live.dsv', 'text/dsv', 'lat;lng;value', '37;-122;1', ['lat', 'lng', 'value']],
+ [
+ 'live',
+ 'text/tab-separated-values',
+ 'lat\tlng\tvalue',
+ '37\t-122\t1',
+ ['lat', 'lng', 'value']
+ ],
+ ['live.csv', 'text/csv', '2025,2024,value', '1,2,3', ['2024', '2025', 'value']]
+ ];
+ try {
+ for (const [name, mimeType, header, row, expectedNames] of fixtures) {
+ let body = `${header}\n${row}\n`;
+ global.fetch = async () => new Response(body, {headers: {'content-type': mimeType}});
+ const source = `https://example.com/${name}`;
+ const loaded = await loadExternallyHostedDataset({source});
+ const csvHeader = loaded.data.fields.map(field => field.name);
+ t.deepEqual(csvHeader, expectedNames, 'establishes columns using the normal import path');
+ body = `${header}\n`;
+ try {
+ const result = await loadExternallyHostedDataset({source, csvHeader});
+ t.deepEqual(result.data, {fields: [], rows: []}, 'recognizes the unchanged empty snapshot');
+ } catch (error) {
+ t.fail(`unchanged header-only ${name} rejected: ${error.message}`);
+ }
+ }
+ } finally {
+ global.fetch = originalFetch;
+ }
+ t.end();
+});
+
test('#remote-file -> getExtensionFromUrl', t => {
t.equal(getExtensionFromUrl('https://example.com/data.geojson'), 'geojson');
t.equal(
diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.js
index affc64602..540c63471 100644
--- a/test/node/reducers/vis-state-test.js
+++ b/test/node/reducers/vis-state-test.js
@@ -31,6 +31,7 @@ import {
processCsvData,
processGeojson,
processArrowBatches,
+ loadExternallyHostedDataset,
remoteDatasetFromUrl
} from '@kepler.gl/processors';
import {
@@ -10154,6 +10155,117 @@ test('VisStateUpdater -> refreshDataset', async t => {
t.end();
});
+test('VisStateUpdater -> CSV refresh preserves layers and filters through 100, 1, 0 and 100 rows', async t => {
+ const originalFetch = global.fetch;
+ const source = 'https://example.com/live.csv';
+ const populated = `lat,lng,value\n${Array.from(
+ {length: 100},
+ (_, i) => `${37 + i / 1000},${-122 - i / 1000},${i}`
+ ).join('\n')}\n`;
+ let body = populated;
+ global.fetch = async () => new Response(body, {headers: {'content-type': 'text/csv'}});
+ const refresh = async state => {
+ drainTasksForTesting();
+ state = reducer(state, VisStateActions.refreshDataset('remote-csv'));
+ const [task] = drainTasksForTesting();
+ const action = await new Promise(resolve => {
+ task.run((effect, succeed, fail, ctx) => effect(succeed, fail, ctx), resolve, resolve, {
+ onProgress: progress => {
+ state = reducer(state, progress);
+ }
+ });
+ });
+ return reducer(state, action);
+ };
+ try {
+ for (const gpu of [false, true]) {
+ const inverted = !gpu;
+ body = populated;
+ const loaded = await loadExternallyHostedDataset({source, format: 'csv'});
+ const datasets = await createNewDataEntryMock({
+ info: {id: 'remote-csv', type: DatasetType.EXTERNALLY_HOSTED},
+ data: loaded.data,
+ metadata: {source, sourceFormat: 'csv'}
+ });
+ const {props} = PointLayer.findDefaultLayerProps(datasets['remote-csv']);
+ const point = new PointLayer({id: 'csv-point', dataId: 'remote-csv', ...props[0]});
+ point.updateLayerVisConfig({opacity: 0.37, radius: 9});
+ let state = {...INITIAL_VIS_STATE, datasets, layers: [point], layerData: [{}]};
+ const initial = calculateLayerData(point, state);
+ state = {...state, layers: [initial.layer], layerData: [initial.layerData]};
+ const style = {...state.layers[0].config.visConfig};
+ const types = datasets['remote-csv'].fields.map(field => field.type);
+ state = reducer(state, VisStateActions.addFilter('remote-csv'));
+ state = reducer(state, VisStateActions.setFilter(0, 'name', 'value'));
+ state = reducer(state, VisStateActions.setFilter(0, 'value', [20, 80]));
+ state = reducer(state, VisStateActions.setFilter(0, 'inverted', inverted));
+ state = reducer(state, VisStateActions.setFilter(0, 'gpu', gpu));
+ const filterId = state.filters[0].id;
+
+ for (const [text, count] of [
+ [`lat,lng,value\n40.5,-125.5,${gpu ? 50 : 10}\n`, 1],
+ ['lat,lng,value\n', 0],
+ [populated, 100]
+ ]) {
+ body = text;
+ state = await refresh(state);
+ const ds = state.datasets['remote-csv'];
+ t.equal(ds.metadata.refreshStatus, 'idle', 'refresh succeeds');
+ t.equal(ds.dataContainer.numRows(), count, 'replaces all rows');
+ t.deepEqual(
+ ds.fields.map(field => field.type),
+ types,
+ 'preserves established types'
+ );
+ t.equal(state.layers.length, 1, 'keeps the configured layer');
+ t.equal(state.layers[0].id, 'csv-point', 'keeps layer identity');
+ t.equal(state.layers[0].isValid, true, 'keeps the layer valid');
+ t.deepEqual(state.layers[0].config.visConfig, style, 'keeps user styling');
+ t.equal(state.filters[0].id, filterId, 'keeps filter identity');
+ t.deepEqual(state.filters[0].value, [20, 80], 'keeps selected range');
+ t.equal(state.filters[0].inverted, inverted, 'keeps Filter out');
+ t.equal(state.filters[0].gpu, gpu, 'keeps evaluation mode');
+ ds.filterTableCPU(state.filters, state.layers);
+ const expected = ds.allIndexes.filter(i => {
+ const value = ds.dataContainer.valueAt(i, 2);
+ return inverted ? value < 20 || value > 80 : value >= 20 && value <= 80;
+ });
+ t.deepEqual(ds.filteredIdxCPU, expected, 'export filtering uses the new snapshot');
+ t.equal(
+ state.layerData[0].data.length,
+ gpu ? count : expected.length,
+ 'rebuilds layer data'
+ );
+ if (count === 1) {
+ t.deepEqual(
+ state.layerData[0].data[0].position.slice(0, 2),
+ [-125.5, 40.5],
+ 'updates coordinates'
+ );
+ }
+ }
+ for (const invalid of ['', 'lat,lng,"value', 'value,lng,lat\n']) {
+ body = invalid;
+ const previous = state.datasets['remote-csv'].dataContainer;
+ const layerData = state.layerData[0];
+ state = await refresh(state);
+ t.equal(
+ state.datasets['remote-csv'].metadata.refreshStatus,
+ 'error',
+ 'unsupported refresh fails'
+ );
+ t.equal(state.datasets['remote-csv'].dataContainer, previous, 'failure retains the rows');
+ t.equal(state.layerData[0], layerData, 'failure retains rendered data');
+ t.deepEqual(state.filters[0].value, [20, 80], 'failure retains the selection');
+ }
+ }
+ } finally {
+ global.fetch = originalFetch;
+ drainTasksForTesting();
+ }
+ t.end();
+});
+
test('VisStateUpdater -> refreshDataset keeps Arrow point layers valid through empty snapshots', async t => {
const snapshot = values => {
const table = arrow.tableFromArrays({