Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
fix(table): avoid partial updates for large row upserts (#3974)
Igor Dykhtacdeccd35c7c2
diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.tsindex e06fa5232..c9b63470d 100644--- a/src/table/src/kepler-table.ts+++ b/src/table/src/kepler-table.ts@@ -457,12 +457,14 @@ class KeplerTable<F extends Field = Field> {return false;}} else {- for (const [index, row] of replacements) {- this.dataContainer.replace(index, row);- }+ // Append can reject or throw. Do it before overwriting existing rows so a+ // failed append leaves those rows and their cached state untouched.if (toAppend.length && !this.dataContainer.append(toAppend)) {return false;}+ for (const [index, row] of replacements) {+ this.dataContainer.replace(index, row);+ }}const nextIndexes =diff --git a/src/utils/src/row-data-container.ts b/src/utils/src/row-data-container.tsindex f595780f6..b6b8a6b45 100644--- a/src/utils/src/row-data-container.ts+++ b/src/utils/src/row-data-container.ts@@ -83,7 +83,11 @@ export class RowDataContainer implements DataContainerInterface {}}- this._rows.push(...rows);+ // Keep the batch size bounded by iteration, not the engine's argument limit.+ const count = rows.length;+ for (let i = 0; i < count; i++) {+ this._rows.push(rows[i]);+ }this._numColumns = expected;return true;}diff --git a/test/node/utils/data-container-test.js b/test/node/utils/data-container-test.jsindex 1161c9481..139cba359 100644--- a/test/node/utils/data-container-test.js+++ b/test/node/utils/data-container-test.js@@ -168,6 +168,19 @@ test('RowDataContainer.update replaces rows', t => {t.end();});+test('RowDataContainer.append accepts a large batch without copying existing rows', t => {+ const first = [0, 10];+ const rows = [first];+ const dc = createDataContainer(rows);+ const incoming = Array.from({length: 200001}, (_, i) => [i + 1, i]);+ t.doesNotThrow(() => t.ok(dc.append(incoming), 'accepts all rows'), 'no argument limit');+ t.equal(dc.flattenData(), rows, 'retains the backing array');+ t.equal(dc.rowAsArray(0), first, 'retains existing row references');+ t.equal(dc.numRows(), 200002, 'includes every appended row');+ t.equal(dc.rowAsArray(200001), incoming[200000], 'retains the last incoming row');+ t.end();+});+test('RowDataContainer.append and remove', t => {const dc = createDataContainer([[10, 20],diff --git a/test/node/utils/kepler-table-test.js b/test/node/utils/kepler-table-test.jsindex d5025bb3a..d6731d4ac 100644--- a/test/node/utils/kepler-table-test.js+++ b/test/node/utils/kepler-table-test.js@@ -912,6 +912,83 @@ test('KeplerTable -> Arrow upserts match encoded Date and timestamp keys', t =>t.end();});+test('KeplerTable -> large row upserts commit rows, indexes and cached domains together', t => {+ const untouched = ['keep', -1];+ const table = new KeplerTable({info: {id: 'large-upsert'}, color: [0, 0, 0]}).updateSchema({+ fields: [+ {name: 'id', type: 'string'},+ {name: 'value', type: 'integer'}+ ],+ rows: [['existing', 0], untouched]+ });+ const container = table.dataContainer;+ const rows = container.flattenData();+ const revision = table.dataRevision;+ const accessor = table.fields[1].valueAccessor;+ table.getColumnFilterProps('value');+ table.filterTableCPU([], []);+ table.sortColumn = 'value';+ table.sortOrder = [1, 0];+ const incoming = Array.from({length: 200001}, (_, i) => [`new-${i}`, i + 1]);+ incoming.unshift(['existing', 2]);+ incoming.push(['existing', 3], ['new-0', 4]);++ t.doesNotThrow(() => t.ok(table.upsertRows(incoming, 'id'), 'accepts the batch'));+ t.equal(table.dataContainer, container, 'retains the container');+ t.equal(container.flattenData(), rows, 'retains the backing array');+ t.equal(container.rowAsArray(1), untouched, 'retains untouched row references');+ t.equal(container.numRows(), 200003, 'appends each new key once');+ t.deepEqual(container.rowAsArray(0), ['existing', 3], 'last existing key wins');+ t.deepEqual(container.rowAsArray(2), ['new-0', 4], 'last new key wins in first-seen order');+ t.deepEqual(container.rowAsArray(200002), ['new-200000', 200001], 'includes the final row');+ t.equal(accessor({index: 0}), 3, 'existing accessors read the committed rows');+ t.equal(table.dataRevision, revision + 1, 'increments revision once');+ t.equal(table.allIndexes.length, 200003, 'updates the index count');+ t.equal(table.allIndexes[200002], 200002, 'indexes the final row');+ t.equal(table.filteredIndex, table.allIndexes, 'resets filtered indexes');+ t.equal(table.filteredIdxCPU, undefined, 'invalidates cached export filtering');+ t.equal(table.sortOrder, null, 'invalidates cached sorting');+ t.deepEqual(table.getColumnFilterProps('value').domain, [-1, 200001], 'refreshes cached domain');+ t.end();+});++test('KeplerTable -> rejected row upserts preserve rows and cached state', t => {+ for (const failure of ['validation', 'append']) {+ const original = ['existing', 1];+ const rows = [original];+ const table = new KeplerTable({info: {id: 'rejected-upsert'}, color: [0, 0, 0]}).updateSchema({+ fields: [+ {name: 'id', type: 'string'},+ {name: 'value', type: 'integer'}+ ],+ rows+ });+ const revision = table.dataRevision;+ const indexes = table.allIndexes;+ const props = table.getColumnFilterProps('value');+ table.filterTableCPU([], []);+ const filtered = table.filteredIdxCPU;+ const incoming = [+ ['existing', 2],+ ['new', 3]+ ];+ if (failure === 'validation') {+ incoming.push(['invalid-width']);+ t.equal(table.upsertRows(incoming, 'id'), false, 'rejects a later invalid row');+ } else {+ Object.preventExtensions(rows);+ t.throws(() => table.upsertRows(incoming, 'id'), 'append to a non-extensible array fails');+ }+ t.equal(table.dataContainer.rowAsArray(0), original, `${failure} leaves original row intact`);+ t.equal(table.length, 1, `${failure} leaves row count intact`);+ t.equal(table.dataRevision, revision, `${failure} leaves revision intact`);+ t.equal(table.allIndexes, indexes, `${failure} leaves indexes intact`);+ t.equal(table.filteredIdxCPU, filtered, `${failure} leaves export filtering intact`);+ t.equal(table.getColumnFilterProps('value'), props, `${failure} leaves cached domain intact`);+ }+ t.end();+});+test('KeplerTable -> row upserts keep distinct number and string keys', t => {const table = new KeplerTable({info: {id: 'row-keys'}, color: [0, 0, 0]}).updateSchema({fields: [