Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

fix(table): avoid partial updates for large row upserts (#3974)

Igor Dykhtacdeccd35c7c2
diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.ts
index e06fa5232..c9b63470d 100644
--- a/src/table/src/kepler-table.ts
+++ b/src/table/src/kepler-table.ts
@@ -457,12 +457,14 @@ class KeplerTable<F extends Field = Field> {
return false;
}
} else {
- for (const [index, row] of replacements) {
- this.dataContainer.replace(index, row);
- }
+ // Append can reject or throw. Do it before overwriting existing rows so a
+ // failed append leaves those rows and their cached state untouched.
if (toAppend.length && !this.dataContainer.append(toAppend)) {
return false;
}
+ for (const [index, row] of replacements) {
+ this.dataContainer.replace(index, row);
+ }
}
const nextIndexes =
diff --git a/src/utils/src/row-data-container.ts b/src/utils/src/row-data-container.ts
index f595780f6..b6b8a6b45 100644
--- a/src/utils/src/row-data-container.ts
+++ b/src/utils/src/row-data-container.ts
@@ -83,7 +83,11 @@ export class RowDataContainer implements DataContainerInterface {
}
}
- this._rows.push(...rows);
+ // Keep the batch size bounded by iteration, not the engine's argument limit.
+ const count = rows.length;
+ for (let i = 0; i < count; i++) {
+ this._rows.push(rows[i]);
+ }
this._numColumns = expected;
return true;
}
diff --git a/test/node/utils/data-container-test.js b/test/node/utils/data-container-test.js
index 1161c9481..139cba359 100644
--- a/test/node/utils/data-container-test.js
+++ b/test/node/utils/data-container-test.js
@@ -168,6 +168,19 @@ test('RowDataContainer.update replaces rows', t => {
t.end();
});
+test('RowDataContainer.append accepts a large batch without copying existing rows', t => {
+ const first = [0, 10];
+ const rows = [first];
+ const dc = createDataContainer(rows);
+ const incoming = Array.from({length: 200001}, (_, i) => [i + 1, i]);
+ t.doesNotThrow(() => t.ok(dc.append(incoming), 'accepts all rows'), 'no argument limit');
+ t.equal(dc.flattenData(), rows, 'retains the backing array');
+ t.equal(dc.rowAsArray(0), first, 'retains existing row references');
+ t.equal(dc.numRows(), 200002, 'includes every appended row');
+ t.equal(dc.rowAsArray(200001), incoming[200000], 'retains the last incoming row');
+ t.end();
+});
+
test('RowDataContainer.append and remove', t => {
const dc = createDataContainer([
[10, 20],
diff --git a/test/node/utils/kepler-table-test.js b/test/node/utils/kepler-table-test.js
index d5025bb3a..d6731d4ac 100644
--- a/test/node/utils/kepler-table-test.js
+++ b/test/node/utils/kepler-table-test.js
@@ -912,6 +912,83 @@ test('KeplerTable -> Arrow upserts match encoded Date and timestamp keys', t =>
t.end();
});
+test('KeplerTable -> large row upserts commit rows, indexes and cached domains together', t => {
+ const untouched = ['keep', -1];
+ const table = new KeplerTable({info: {id: 'large-upsert'}, color: [0, 0, 0]}).updateSchema({
+ fields: [
+ {name: 'id', type: 'string'},
+ {name: 'value', type: 'integer'}
+ ],
+ rows: [['existing', 0], untouched]
+ });
+ const container = table.dataContainer;
+ const rows = container.flattenData();
+ const revision = table.dataRevision;
+ const accessor = table.fields[1].valueAccessor;
+ table.getColumnFilterProps('value');
+ table.filterTableCPU([], []);
+ table.sortColumn = 'value';
+ table.sortOrder = [1, 0];
+ const incoming = Array.from({length: 200001}, (_, i) => [`new-${i}`, i + 1]);
+ incoming.unshift(['existing', 2]);
+ incoming.push(['existing', 3], ['new-0', 4]);
+
+ t.doesNotThrow(() => t.ok(table.upsertRows(incoming, 'id'), 'accepts the batch'));
+ t.equal(table.dataContainer, container, 'retains the container');
+ t.equal(container.flattenData(), rows, 'retains the backing array');
+ t.equal(container.rowAsArray(1), untouched, 'retains untouched row references');
+ t.equal(container.numRows(), 200003, 'appends each new key once');
+ t.deepEqual(container.rowAsArray(0), ['existing', 3], 'last existing key wins');
+ t.deepEqual(container.rowAsArray(2), ['new-0', 4], 'last new key wins in first-seen order');
+ t.deepEqual(container.rowAsArray(200002), ['new-200000', 200001], 'includes the final row');
+ t.equal(accessor({index: 0}), 3, 'existing accessors read the committed rows');
+ t.equal(table.dataRevision, revision + 1, 'increments revision once');
+ t.equal(table.allIndexes.length, 200003, 'updates the index count');
+ t.equal(table.allIndexes[200002], 200002, 'indexes the final row');
+ t.equal(table.filteredIndex, table.allIndexes, 'resets filtered indexes');
+ t.equal(table.filteredIdxCPU, undefined, 'invalidates cached export filtering');
+ t.equal(table.sortOrder, null, 'invalidates cached sorting');
+ t.deepEqual(table.getColumnFilterProps('value').domain, [-1, 200001], 'refreshes cached domain');
+ t.end();
+});
+
+test('KeplerTable -> rejected row upserts preserve rows and cached state', t => {
+ for (const failure of ['validation', 'append']) {
+ const original = ['existing', 1];
+ const rows = [original];
+ const table = new KeplerTable({info: {id: 'rejected-upsert'}, color: [0, 0, 0]}).updateSchema({
+ fields: [
+ {name: 'id', type: 'string'},
+ {name: 'value', type: 'integer'}
+ ],
+ rows
+ });
+ const revision = table.dataRevision;
+ const indexes = table.allIndexes;
+ const props = table.getColumnFilterProps('value');
+ table.filterTableCPU([], []);
+ const filtered = table.filteredIdxCPU;
+ const incoming = [
+ ['existing', 2],
+ ['new', 3]
+ ];
+ if (failure === 'validation') {
+ incoming.push(['invalid-width']);
+ t.equal(table.upsertRows(incoming, 'id'), false, 'rejects a later invalid row');
+ } else {
+ Object.preventExtensions(rows);
+ t.throws(() => table.upsertRows(incoming, 'id'), 'append to a non-extensible array fails');
+ }
+ t.equal(table.dataContainer.rowAsArray(0), original, `${failure} leaves original row intact`);
+ t.equal(table.length, 1, `${failure} leaves row count intact`);
+ t.equal(table.dataRevision, revision, `${failure} leaves revision intact`);
+ t.equal(table.allIndexes, indexes, `${failure} leaves indexes intact`);
+ t.equal(table.filteredIdxCPU, filtered, `${failure} leaves export filtering intact`);
+ t.equal(table.getColumnFilterProps('value'), props, `${failure} leaves cached domain intact`);
+ }
+ t.end();
+});
+
test('KeplerTable -> row upserts keep distinct number and string keys', t => {
const table = new KeplerTable({info: {id: 'row-keys'}, color: [0, 0, 0]}).updateSchema({
fields: [