Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

39.7 KB
Raw Blame
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import test from 'tape';
import moment from 'moment';
import testData, {numericRangesCsv, testFields} from 'test/fixtures/test-csv-data';

import {preciseRound, getFilterFunction} from '@kepler.gl/utils';
import {
  findPointFieldPairs,
  KeplerTable,
  createNewDataEntry,
  joinDatasets,
  sortDatasetByColumn
} from '@kepler.gl/table';
import {processArrowBatches, processCsvData} from '@kepler.gl/processors';
import {ALL_FIELD_TYPES, FILTER_TYPES} from '@kepler.gl/constants';
import * as arrow from 'apache-arrow';

import {cmpFields} from '../../helpers/comparison-utils';
import {createNewDataEntryMock} from '../../helpers/table-utils';

function testGetTimeFieldDomain(table, t) {
  const test_cases = [
    {
      name: 'default',
      input: table.getColumnFilterDomain(table.fields[0]).domain,
      output: [
        moment.utc('2016-09-17 00:09:55').valueOf(),
        moment.utc('2016-09-17 00:30:08').valueOf()
      ],
      msg: '2016-09-17 00:30:08'
    },
    {
      name: 'epoch',
      input: table.getColumnFilterDomain(table.fields[4]).domain,
      output: [moment.utc(1472688000000).valueOf(), moment.utc(1472774400000).valueOf()],
      msg: 1472688000000
    },
    {
      name: 'T',
      input: table.getColumnFilterDomain(table.fields[7]).domain,
      output: [
        moment.utc('2016-09-23T00:00:00.000Z').valueOf(),
        moment.utc('2016-09-23T08:00:00.000Z').valueOf()
      ],
      msg: '2016-09-23T00:00:00.000Z'
    },
    {
      name: 'UTC',
      input: table.getColumnFilterDomain(table.fields[8]).domain,
      output: [
        moment.utc('2016-10-01 09:41:39+00:00').valueOf(),
        moment.utc('2016-10-01 10:01:54+00:00').valueOf()
      ],
      msg: '2016-10-01 09:41:39+00:00'
    },
    {
      name: 'local',
      input: table.getColumnFilterDomain(table.fields[9]).domain,
      output: [
        moment.utc('2016-10-01 09:41:39+00:00').valueOf(),
        moment.utc('2016-10-01 17:01:54+00:00').valueOf()
      ],
      msg: '2016-10-01 09:41:39+00:00'
    }
  ];

  test_cases.forEach(tc =>
    t.deepEqual(tc.input, tc.output, `should process correct domain for timestamp ${tc.msg}`)
  );
}

function testGetNumericFieldStep(table, t) {
  const test_cases = [
    {
      name: 'smallest',
      input: table.getColumnFilterDomain(table.fields[0]).step,
      output: 0.0000001
    },
    {
      name: 'small',
      input: table.getColumnFilterDomain(table.fields[1]).step,
      output: 0.001
    },
    {
      name: 'negative',
      input: table.getColumnFilterDomain(table.fields[2]).step,
      output: 0.01
    },
    {
      name: 'medium',
      input: table.getColumnFilterDomain(table.fields[3]).step,
      output: 0.01
    },
    {
      name: 'large',
      input: table.getColumnFilterDomain(table.fields[4]).step,
      output: 1
    }
  ];

  test_cases.forEach(tc =>
    t.equal(
      preciseRound(tc.input, 5),
      preciseRound(tc.output, 5),
      `should process correct step for field ${tc.name}`
    )
  );
}

function testGetFilterFunction({fields, dataContainer}, t) {
  const dataId = 'dataset-1';
  const timeStringFilter = {
    fieldIdx: [0],
    type: FILTER_TYPES.timeRange,
    value: [
      moment.utc('2016-09-17 00:09:55').valueOf(),
      moment.utc('2016-09-17 00:20:08').valueOf()
    ],
    id: 'filter-1',
    dataId: [dataId]
  };

  let field = fields[timeStringFilter.fieldIdx[0]];

  let filterFunction = getFilterFunction(field, dataId, timeStringFilter, [], dataContainer);

  t.equal(
    filterFunction({index: 10}),
    true,
    `${dataContainer.valueAt(10, 0)} should be inside the range`
  );

  t.equal(
    filterFunction({index: 15}),
    false,
    `${dataContainer.valueAt(15, 0)} should be outside the range`
  );

  const epochFilter = {
    fieldIdx: [4],
    type: FILTER_TYPES.timeRange,
    value: [moment.utc(1472688000000).valueOf(), moment.utc(1472734400000).valueOf()],
    id: 'filter-2',
    dataId: [dataId]
  };

  field = fields[epochFilter.fieldIdx[0]];

  filterFunction = getFilterFunction(field, dataId, epochFilter, [], dataContainer);

  t.equal(
    filterFunction({index: 10}),
    true,
    `${dataContainer.valueAt(10, 1)} should be inside the range`
  );

  t.equal(
    filterFunction({index: 15}),
    false,
    `${dataContainer.valueAt(15, 1)} should be outside the range`
  );

  const tzFilter = {
    fieldIdx: [7],
    type: FILTER_TYPES.timeRange,
    value: [
      moment.utc('2016-09-23T00:00:00.000Z').valueOf(),
      moment.utc('2016-09-23T06:00:00.000Z').valueOf()
    ],
    id: 'filter-3',
    dataId: [dataId]
  };

  field = fields[tzFilter.fieldIdx[0]];

  filterFunction = getFilterFunction(field, dataId, tzFilter, [], dataContainer);

  t.equal(
    filterFunction({index: 10}),
    true,
    `${dataContainer.valueAt(10, 7)} should be inside the range`
  );

  t.equal(
    filterFunction({index: 23}),
    false,
    `${dataContainer.valueAt(23, 7)} should be outside the range`
  );

  const utcFilter = {
    fieldIdx: [8],
    type: FILTER_TYPES.timeRange,
    value: [
      moment.utc('2016-10-01 09:45:39+00:00').valueOf(),
      moment.utc('2016-10-01 10:00:00+00:00').valueOf()
    ],
    id: 'filter-4',
    dataId: [dataId]
  };

  field = fields[utcFilter.fieldIdx[0]];

  filterFunction = getFilterFunction(field, dataId, utcFilter, [], dataContainer);

  t.equal(
    filterFunction({index: 6}),
    false,
    `${dataContainer.valueAt(0, 8)} should be outside the range`
  );

  t.equal(
    filterFunction({index: 4}),
    true,
    `${dataContainer.valueAt(4, 8)} should be inside the range`
  );

  t.equal(
    filterFunction({index: 23}),
    false,
    `${dataContainer.valueAt(23, 8)} should be outside the range`
  );
}

test('KeplerTable -> getColumnFilterDomain -> time', async t => {
  const expectedFields = testFields;

  const data = processCsvData(testData);
  const dataset = (
    await createNewDataEntryMock({
      info: {id: 'test'},
      data
    })
  ).test;
  cmpFields(t, expectedFields, dataset.fields, dataset.id);
  testGetTimeFieldDomain(dataset, t);
  testGetFilterFunction(dataset, t);

  t.end();
});

test('KeplerTable -> getColumnFilterDomain -> numeric', async t => {
  const data = processCsvData(numericRangesCsv);
  const dataset = (
    await createNewDataEntryMock({
      info: {id: 'test'},
      data
    })
  ).test;

  testGetNumericFieldStep(dataset, t);

  t.end();
});

test('KeplerTable -> findPointFieldPairs', t => {
  const TASE_CASE = [
    {
      fields: [
        'point-lat',
        'point-lng',
        'long',
        'lat',
        'poi_latitude',
        'poi_longitude',
        'latino',
        'lngtino',
        'lat.1',
        'lng.1'
      ],
      expected: [
        {
          defaultName: 'point',
          pair: {
            lat: {
              fieldIdx: 0,
              value: 'point-lat'
            },
            lng: {
              fieldIdx: 1,
              value: 'point-lng'
            }
          },
          suffix: ['lat', 'lng']
        },
        {
          defaultName: 'point',
          pair: {
            lat: {
              fieldIdx: 3,
              value: 'lat'
            },
            lng: {
              fieldIdx: 2,
              value: 'long'
            }
          },
          suffix: ['lat', 'long']
        },
        {
          defaultName: 'poi',
          pair: {
            lat: {
              fieldIdx: 4,
              value: 'poi_latitude'
            },
            lng: {
              fieldIdx: 5,
              value: 'poi_longitude'
            }
          },
          suffix: ['latitude', 'longitude']
        },
        {
          defaultName: '1',
          pair: {
            lat: {
              fieldIdx: 8,
              value: 'lat.1'
            },
            lng: {
              fieldIdx: 9,
              value: 'lng.1'
            }
          },
          suffix: ['lat', 'lng']
        }
      ]
    },
    {
      fields: ['point.lat', 'point.long', 'point.altitude', 'latitude', 'longitude'],
      expected: [
        {
          defaultName: 'point',
          pair: {
            lat: {
              fieldIdx: 0,
              value: 'point.lat'
            },
            lng: {
              fieldIdx: 1,
              value: 'point.long'
            },
            altitude: {
              fieldIdx: 2,
              value: 'point.altitude'
            }
          },
          suffix: ['lat', 'long']
        },
        {
          defaultName: 'point',
          pair: {
            lat: {
              fieldIdx: 3,
              value: 'latitude'
            },
            lng: {
              fieldIdx: 4,
              value: 'longitude'
            }
          },
          suffix: ['latitude', 'longitude']
        }
      ]
    },
    {
      fields: ['point_lat', 'point_lng', 'alt'],
      expected: [
        {
          defaultName: 'point',
          pair: {
            // no matching "alt" altitude found for this pair
            lat: {
              fieldIdx: 0,
              value: 'point_lat'
            },
            lng: {
              fieldIdx: 1,
              value: 'point_lng'
            }
          },
          suffix: ['lat', 'lng']
        }
      ]
    },
    {
      fields: ['point_lat', 'point_lng', 'point_alt'],
      expected: [
        {
          defaultName: 'point',
          pair: {
            // a matching "point_alt" altitude was found for this pair
            lat: {
              fieldIdx: 0,
              value: 'point_lat'
            },
            lng: {
              fieldIdx: 1,
              value: 'point_lng'
            },
            altitude: {
              fieldIdx: 2,
              value: 'point_alt'
            }
          },
          suffix: ['lat', 'lng']
        }
      ]
    }
  ];

  TASE_CASE.forEach(({fields, expected}) => {
    const found = findPointFieldPairs(fields.map(f => ({name: f})));

    t.equal(expected.length, found.length, `should found ${expected.length} pairs`);
    expected.forEach((pair, index) => {
      t.deepEqual(found[index], pair, 'should found correct point pair');
    });
  });

  t.end();
});

test('KeplerTable -> Int64 field accessor converts BigInt to number', async t => {
  const arrowTable = arrow.tableFromArrays({
    count: new BigInt64Array([1n, 9999n]),
    name: ['a', 'b']
  });
  const table = new KeplerTable({info: {id: 'int64'}, color: [0, 0, 0]});
  await table.importData({
    data: {
      fields: [
        {name: 'count', type: ALL_FIELD_TYPES.integer, analyzerType: 'INT'},
        {name: 'name', type: ALL_FIELD_TYPES.string, analyzerType: 'STRING'}
      ],
      cols: [arrowTable.getChildAt(0), arrowTable.getChildAt(1)],
      arrowTable
    }
  });

  t.equal(typeof table.dataContainer.valueAt(0, 0), 'bigint', 'valueAt stays raw BigInt');
  t.equal(table.dataContainer.valueAt(0, 0), 1n);
  t.equal(
    typeof table.fields[0].valueAccessor({index: 0}),
    'number',
    'Int64 valueAccessor returns a JS number'
  );
  t.equal(table.fields[0].valueAccessor({index: 0}), 1);
  t.equal(table.fields[0].valueAccessor({index: 1}), 9999);
  t.equal(table.fields[1].valueAccessor({index: 0}), 'a', 'non-Int64 accessors are unchanged');

  t.end();
});

test('KeplerTable -> ordinary row access leaves numbers and strings unchanged', t => {
  const rows = [
    [10, 0.5],
    ['9007199254740993', '12'],
    [null, undefined]
  ];
  const table = new KeplerTable({color: [0, 0, 0]}).updateSchema({
    fields: [
      {name: 'integer', type: 'integer'},
      {name: 'real', type: 'real'}
    ],
    rows
  });
  for (const [column, field] of table.fields.entries()) {
    const expected = rows.map(row => row[column]);
    t.deepEqual(
      rows.map((_, index) => field.valueAccessor({index})),
      expected
    );
    t.deepEqual(rows.map(field.valueAccessor), expected, 'materialized rows agree');
  }
  t.deepEqual(table.dataContainer.flattenData(), rows, 'raw storage unchanged');
  t.end();
});

test('KeplerTable -> derived BigInt access preserves exact raw sorting', t => {
  const rows = [[9007199254740993n], [9007199254740992n], [null]];
  const source = new KeplerTable({color: [0, 0, 0]}).updateSchema({
    fields: [{name: 'value', type: 'integer'}],
    rows
  });
  const proto = joinDatasets(source, source, {
    leftField: 'value',
    rightField: 'value',
    type: 'LEFT',
    rightColumns: []
  });
  const table = new KeplerTable({color: [0, 0, 0]}).updateSchema(proto.data);
  t.equal(table.fields[0].valueAccessor({index: 0}), 9007199254740992, 'numeric view can round');
  sortDatasetByColumn(table, 'value', 'ASCENDING');
  t.deepEqual(table.sortOrder, [1, 0, 2], 'sorting distinguishes adjacent unsafe integers');
  t.deepEqual(table.dataContainer.flattenData(), rows, 'sorting leaves exact raw cells intact');
  t.end();
});

test('KeplerTable -> update with arrow cols keeps gpu filter state', async t => {
  const first = arrow.tableFromArrays({
    lat: [1, 2],
    lng: [3, 4]
  });
  const table = new KeplerTable({info: {id: 'arrow-inc'}, color: [0, 0, 0]});
  await table.importData({
    data: {
      fields: [
        {name: 'lat', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'},
        {name: 'lng', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'}
      ],
      rows: [],
      cols: [first.getChildAt(0), first.getChildAt(1)],
      arrowTable: first
    }
  });

  const filters = [
    {
      id: 'keep-one',
      dataId: [table.id],
      name: ['lat'],
      fieldIdx: [0],
      type: FILTER_TYPES.range,
      domain: [1, 3],
      value: [1, 1],
      enabled: true,
      gpu: true,
      gpuChannel: [0]
    }
  ];
  table.filterTable(filters, []);
  table.filterTableCPU(filters, []);
  t.deepEqual(table.filteredIdxCPU, [0], 'initial CPU evaluation keeps the matching row');
  const gpuFilter = table.gpuFilter;

  const next = arrow.tableFromArrays({
    lat: [3, 1, 2],
    lng: [3, 4, 5]
  });
  await table.update({
    fields: [
      {name: 'lat', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'},
      {name: 'lng', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'}
    ],
    rows: [],
    cols: [next.getChildAt(0), next.getChildAt(1)],
    arrowTable: next
  });

  t.equal(table.length, 3, 'arrow incremental update should grow rows');
  t.equal(table.gpuFilter, gpuFilter, 'arrow incremental update should keep gpuFilter');
  t.equal(table.dataRevision, 1, 'arrow incremental update should bump dataRevision');
  table.filterTable(filters, []);
  table.filterTableCPU(filters, []);
  t.deepEqual(table.filteredIndex, [0, 1, 2], 'GPU filtering keeps all rows available to the GPU');
  t.deepEqual(table.filteredIndexForDomain, [1], 'domain filtering uses the refreshed values');
  t.deepEqual(table.filteredIdxCPU, [1], 'CPU evaluation uses the refreshed matching row');
  t.end();
});

test('KeplerTable -> Arrow snapshots rebuild cached timestamps and reuse them between filters', async t => {
  const firstTime = Date.UTC(2024, 1, 1);
  const lastTime = firstTime + 86400000;
  const types = [
    arrow.TimestampSecond,
    arrow.TimestampMillisecond,
    arrow.TimestampMicrosecond,
    arrow.TimestampNanosecond
  ];

  for (const Timestamp of types) {
    for (const [shape, timezone] of [
      [null, null],
      ['table', 'UTC'],
      ['duckdb', 'America/New_York']
    ]) {
      const snapshot = values => {
        const vector = offset =>
          arrow.vectorFromArray(
            values.map(value => (value === null ? null : new Date(value + offset))),
            new Timestamp(timezone)
          );
        const source = new arrow.Table({
          time: vector(0),
          later: vector(3600000),
          unused: vector(0)
        });
        const ipc = arrow.tableFromIPC(arrow.tableToIPC(source, 'file'));
        const data = processArrowBatches(ipc.batches);
        return shape === 'table' ? {...data, arrowTable: ipc} : data;
      };

      for (const gpu of [false, true]) {
        const table = new KeplerTable({info: {id: 'timestamps'}, color: [0, 0, 0]});
        await table.importData({data: snapshot([firstTime, lastTime, null])});
        const filters = ['time', 'later'].map((name, index) => ({
          ...table.getColumnFilterProps(name),
          id: name,
          dataId: [table.id],
          name: [name],
          fieldIdx: [index],
          value: [firstTime + index * 3600000, firstTime + index * 3600000],
          enabled: true,
          gpu,
          gpuChannel: [index]
        }));
        table.filterTable(filters, []);
        table.filterTableCPU(filters, []);
        t.deepEqual(table.filteredIdxCPU, [0], 'initial timestamp selection');

        const reads = [0, 0, 0];
        table.fields.forEach((field, index) => {
          const accessor = field.valueAccessor;
          field.valueAccessor = datum => {
            reads[index] += 1;
            return accessor(datum);
          };
        });
        for (const values of [
          [lastTime, firstTime, null],
          [],
          [null, lastTime, firstTime],
          [firstTime, lastTime, null, firstTime]
        ]) {
          const data = snapshot(values);
          const oldGpuFilter = table.gpuFilter;
          reads.fill(0);
          await table.update(shape === 'duckdb' ? {...data, fields: []} : data);
          t.equal(table.gpuFilter, oldGpuFilter, 'snapshot retains GPU configuration');
          t.deepEqual(
            reads,
            [values.length, values.length, 0],
            'only cached timestamps are scanned once'
          );
          t.notOk(table.fields[2].filterProps, 'unused timestamp stays lazy');
          t.deepEqual(
            table.fields[0].filterProps.mappedValue,
            values,
            'mapping follows row order and nulls'
          );
          const mapped = table.fields[0].filterProps.mappedValue;
          table.getColumnFilterProps('time');
          table.filterTable(filters, []);
          table.filterTableCPU(filters, []);
          const expected = values.flatMap((value, index) => (value === firstTime ? [index] : []));
          t.deepEqual(table.filteredIdxCPU, expected, 'CPU/export filters use current timestamps');
          const getValue = table.gpuFilter.filterValueAccessor(table.dataContainer)();
          const visible = table.filteredIndex.filter(index =>
            getValue({index}).every((value, channel) => {
              const [min, max] = table.gpuFilter.filterRange[channel];
              return value >= min && value <= max;
            })
          );
          t.deepEqual(visible, expected, 'render filter attributes select the current rows');
          reads.fill(0);
          table.filterTable(filters, []);
          table.filterTableCPU(filters, []);
          t.equal(
            table.fields[0].filterProps.mappedValue,
            mapped,
            'filter interactions reuse the mapping'
          );
          t.deepEqual(reads, [0, 0, 0], 'unchanged filters do not rescan timestamps');
        }
      }
    }
  }
  t.end();
});

test('KeplerTable -> Arrow row edits and snapshots keep timestamp filters current', async t => {
  const firstTime = Date.UTC(2024, 1, 1);
  const lastTime = firstTime + 86400000;
  const snapshot = rows => {
    const source = new arrow.Table({
      id: arrow.vectorFromArray(
        rows.map(row => row[0]),
        new arrow.Utf8()
      ),
      time: arrow.vectorFromArray(
        rows.map(row => row[1]),
        new arrow.TimestampMillisecond()
      )
    });
    return processArrowBatches(source.batches);
  };
  const table = new KeplerTable({info: {id: 'row-edit-timestamps'}, color: [0, 0, 0]});
  await table.importData({
    data: snapshot([
      ['a', firstTime],
      ['b', lastTime]
    ])
  });
  const container = table.dataContainer;
  const accessor = table.fields[1].valueAccessor;
  const filters = [
    {
      ...table.getColumnFilterProps('time'),
      id: 'time',
      dataId: [table.id],
      name: ['time'],
      fieldIdx: [1],
      value: [firstTime, firstTime],
      enabled: true,
      gpu: true,
      gpuChannel: [0]
    }
  ];
  const check = (values, expected) => {
    t.equal(table.dataContainer, container, 'container identity survives edits and refresh');
    t.deepEqual(table.fields[1].filterProps.mappedValue, values, 'timestamp cache follows rows');
    t.deepEqual(
      values.map((_, index) => accessor({index})),
      values,
      'original accessor stays bound'
    );
    table.filterTable(filters, []);
    table.filterTableCPU(filters, []);
    t.deepEqual(table.filteredIdxCPU, expected, 'CPU/export filter follows edited timestamps');
    const getValue = table.gpuFilter.filterValueAccessor(container)();
    const visible = table.filteredIndex.filter(index =>
      getValue({index}).every((value, channel) => {
        const [min, max] = table.gpuFilter.filterRange[channel];
        return value >= min && value <= max;
      })
    );
    t.deepEqual(visible, expected, 'GPU filter follows edited timestamps');
  };
  check([firstTime, lastTime], [0]);
  t.ok(table.appendRows([['c', firstTime]]), 'append succeeds');
  check([firstTime, lastTime, firstTime], [0, 2]);
  t.ok(
    table.upsertRows(
      [
        ['a', lastTime],
        ['b', firstTime]
      ],
      'id'
    ),
    'upsert succeeds'
  );
  check([lastTime, firstTime, firstTime], [1, 2]);
  t.ok(table.removeRows([1]), 'delete succeeds');
  check([lastTime, firstTime], [1]);
  await table.update(
    snapshot([
      ['c', firstTime],
      ['a', lastTime],
      ['d', null]
    ])
  );
  check([firstTime, lastTime, null], [0]);
  t.ok(table.removeRows([0, 1, 2]), 'delete all succeeds');
  check([], []);
  t.ok(table.appendRows([['e', firstTime]]), 'append after empty succeeds');
  check([firstTime], [0]);
  t.end();
});

test('KeplerTable -> Arrow upserts match and deduplicate encoded Int64 keys', t => {
  const existingKey = 9007199254740993n;
  const newKey = existingKey + 1n;
  const source = new arrow.Table({
    id: arrow.vectorFromArray([existingKey], new arrow.Int64()),
    value: arrow.vectorFromArray([10], new arrow.Int32())
  });
  const table = new KeplerTable({info: {id: 'encoded-keys'}, color: [0, 0, 0]}).updateSchema(
    processArrowBatches(source.batches)
  );
  const container = table.dataContainer;
  const snapshot = container.getTable();
  const accessor = table.fields[1].valueAccessor;
  const revision = table.dataRevision;

  t.ok(
    table.upsertRows([[String(existingKey), 20]], 'id'),
    'accepts the JSON decimal-string representation of an Int64 key'
  );
  t.deepEqual(container.flattenData(), [[existingKey, 20]], 'replaces the existing encoded key');
  t.ok(
    table.upsertRows(
      [
        [String(existingKey), 30],
        [String(newKey), 40],
        [existingKey, 50],
        [newKey, 60]
      ],
      'id'
    ),
    'accepts equivalent string and BigInt keys in one edit'
  );
  t.deepEqual(
    container.flattenData(),
    [
      [existingKey, 50],
      [newKey, 60]
    ],
    'last equivalent key wins for both replacements and appended rows'
  );
  t.equal(table.dataContainer, container, 'retains the container');
  t.equal(accessor({index: 0}), 50, 'the original accessor reads the replacement');
  t.equal(snapshot.getChild('value').get(0), 10, 'retains the previous Arrow snapshot');
  t.equal(table.dataRevision, revision + 2, 'increments the revision once per accepted edit');

  const currentSnapshot = container.getTable();
  const currentIndexes = table.allIndexes;
  const currentRevision = table.dataRevision;
  for (const invalidKey of ['invalid-int64', '9223372036854775808', '-9223372036854775809']) {
    t.notOk(
      table.upsertRows(
        [
          [String(existingKey), 70],
          [invalidKey, 80]
        ],
        'id'
      ),
      'an invalid or out-of-range later key rejects the entire edit'
    );
    t.equal(container.getTable(), currentSnapshot, 'rejected keys retain the Arrow snapshot');
    t.equal(table.allIndexes, currentIndexes, 'rejected keys retain the indexes');
    t.equal(table.dataRevision, currentRevision, 'rejected keys retain the revision');
    t.equal(accessor({index: 0}), 50, 'no replacement leaks from the rejected edit');
  }
  t.end();
});

test('KeplerTable -> Arrow upserts reject invalid integer keys before matching', t => {
  const uint64Max = (1n << 64n) - 1n;
  for (const {type, existingKeys, invalidKeys} of [
    {
      type: new arrow.Int8(),
      existingKeys: [-128, 127],
      invalidKeys: [128, '128', -129, 1.5, NaN, Infinity]
    },
    {
      type: new arrow.Uint64(),
      existingKeys: [0n, uint64Max],
      invalidKeys: [uint64Max + 1n, String(uint64Max + 1n), -1n, '-1']
    }
  ]) {
    const source = new arrow.Table({
      id: arrow.vectorFromArray(existingKeys, type),
      value: arrow.vectorFromArray([10, 20], new arrow.Int32())
    });
    const table = new KeplerTable({info: {id: 'integer-keys'}, color: [0, 0, 0]}).updateSchema(
      processArrowBatches(source.batches)
    );
    t.ok(
      table.upsertRows([[String(existingKeys[1]), 30]], 'id'),
      `accepts a numeric string at the ${type} upper boundary`
    );
    const expected = [
      [existingKeys[0], 10],
      [existingKeys[1], 30]
    ];
    t.deepEqual(table.dataContainer.flattenData(), expected, 'matches the stored boundary key');
    const snapshot = table.dataContainer.getTable();
    const revision = table.dataRevision;
    for (const key of invalidKeys) {
      t.notOk(
        table.upsertRows(
          [
            [existingKeys[0], 40],
            [key, 50]
          ],
          'id'
        ),
        `rejects invalid ${type} key ${String(key)} before matching`
      );
      t.equal(table.dataContainer.getTable(), snapshot, 'retains the snapshot');
      t.equal(table.dataRevision, revision, 'retains the revision');
      t.deepEqual(table.dataContainer.flattenData(), expected, 'does not overwrite any row');
    }
  }
  t.end();
});

test('KeplerTable -> Arrow upserts match encoded Date and timestamp keys', t => {
  const first = Date.UTC(2024, 0, 1);
  const next = first + 86400000;
  for (const type of [new arrow.DateMillisecond(), new arrow.TimestampMillisecond()]) {
    const source = new arrow.Table({
      time: arrow.vectorFromArray([first], type),
      value: arrow.vectorFromArray([1], new arrow.Int32())
    });
    const table = new KeplerTable({info: {id: 'date-keys'}, color: [0, 0, 0]}).updateSchema(
      processArrowBatches(source.batches)
    );
    t.ok(
      table.upsertRows(
        [
          [new Date(first), 2],
          [new Date(next), 3],
          [next, 4]
        ],
        'time'
      ),
      `accepts Date keys for ${type}`
    );
    t.deepEqual(
      table.dataContainer.flattenData(),
      [
        [first, 2],
        [next, 4]
      ],
      'matches existing keys and deduplicates equivalent Date and millisecond keys'
    );
  }
  t.end();
});

test('KeplerTable -> large row upserts commit rows, indexes and cached domains together', t => {
  const untouched = ['keep', -1];
  const table = new KeplerTable({info: {id: 'large-upsert'}, color: [0, 0, 0]}).updateSchema({
    fields: [
      {name: 'id', type: 'string'},
      {name: 'value', type: 'integer'}
    ],
    rows: [['existing', 0], untouched]
  });
  const container = table.dataContainer;
  const rows = container.flattenData();
  const revision = table.dataRevision;
  const accessor = table.fields[1].valueAccessor;
  table.getColumnFilterProps('value');
  table.filterTableCPU([], []);
  table.sortColumn = 'value';
  table.sortOrder = [1, 0];
  const incoming = Array.from({length: 200001}, (_, i) => [`new-${i}`, i + 1]);
  incoming.unshift(['existing', 2]);
  incoming.push(['existing', 3], ['new-0', 4]);

  t.doesNotThrow(() => t.ok(table.upsertRows(incoming, 'id'), 'accepts the batch'));
  t.equal(table.dataContainer, container, 'retains the container');
  t.equal(container.flattenData(), rows, 'retains the backing array');
  t.equal(container.rowAsArray(1), untouched, 'retains untouched row references');
  t.equal(container.numRows(), 200003, 'appends each new key once');
  t.deepEqual(container.rowAsArray(0), ['existing', 3], 'last existing key wins');
  t.deepEqual(container.rowAsArray(2), ['new-0', 4], 'last new key wins in first-seen order');
  t.deepEqual(container.rowAsArray(200002), ['new-200000', 200001], 'includes the final row');
  t.equal(accessor({index: 0}), 3, 'existing accessors read the committed rows');
  t.equal(table.dataRevision, revision + 1, 'increments revision once');
  t.equal(table.allIndexes.length, 200003, 'updates the index count');
  t.equal(table.allIndexes[200002], 200002, 'indexes the final row');
  t.equal(table.filteredIndex, table.allIndexes, 'resets filtered indexes');
  t.equal(table.filteredIdxCPU, undefined, 'invalidates cached export filtering');
  t.equal(table.sortOrder, null, 'invalidates cached sorting');
  t.deepEqual(table.getColumnFilterProps('value').domain, [-1, 200001], 'refreshes cached domain');
  t.end();
});

test('KeplerTable -> rejected row upserts preserve rows and cached state', t => {
  for (const failure of ['validation', 'append']) {
    const original = ['existing', 1];
    const rows = [original];
    const table = new KeplerTable({info: {id: 'rejected-upsert'}, color: [0, 0, 0]}).updateSchema({
      fields: [
        {name: 'id', type: 'string'},
        {name: 'value', type: 'integer'}
      ],
      rows
    });
    const revision = table.dataRevision;
    const indexes = table.allIndexes;
    const props = table.getColumnFilterProps('value');
    table.filterTableCPU([], []);
    const filtered = table.filteredIdxCPU;
    const incoming = [
      ['existing', 2],
      ['new', 3]
    ];
    if (failure === 'validation') {
      incoming.push(['invalid-width']);
      t.equal(table.upsertRows(incoming, 'id'), false, 'rejects a later invalid row');
    } else {
      Object.preventExtensions(rows);
      t.throws(() => table.upsertRows(incoming, 'id'), 'append to a non-extensible array fails');
    }
    t.equal(table.dataContainer.rowAsArray(0), original, `${failure} leaves original row intact`);
    t.equal(table.length, 1, `${failure} leaves row count intact`);
    t.equal(table.dataRevision, revision, `${failure} leaves revision intact`);
    t.equal(table.allIndexes, indexes, `${failure} leaves indexes intact`);
    t.equal(table.filteredIdxCPU, filtered, `${failure} leaves export filtering intact`);
    t.equal(table.getColumnFilterProps('value'), props, `${failure} leaves cached domain intact`);
  }
  t.end();
});

test('KeplerTable -> row upserts keep distinct number and string keys', t => {
  const table = new KeplerTable({info: {id: 'row-keys'}, color: [0, 0, 0]}).updateSchema({
    fields: [
      {name: 'id', type: 'string'},
      {name: 'value', type: 'integer'}
    ],
    rows: [[1, 10]]
  });
  t.ok(
    table.upsertRows(
      [
        ['1', 20],
        [1, 30]
      ],
      'id'
    ),
    'accepts row keys without encoding'
  );
  t.deepEqual(
    table.dataContainer.flattenData(),
    [
      [1, 30],
      ['1', 20]
    ],
    'row containers retain exact key equality'
  );
  t.end();
});

test('KeplerTable -> Arrow upserts preserve untouched raw timestamps and nulls', t => {
  const rawValues = vector =>
    vector.data.flatMap(chunk =>
      Array.from({length: chunk.length}, (_, index) =>
        chunk.getValid(index) ? chunk.values[index] : null
      )
    );

  for (const [Timestamp, first, unitsPerMillisecond] of [
    [arrow.TimestampMicrosecond, 9007199254740993n, 1000n],
    [arrow.TimestampNanosecond, 1700000000000000123n, 1000000n]
  ]) {
    const source = new arrow.Table({
      id: arrow.vectorFromArray(['padding', 'a', 'b', 'c', 'd'], new arrow.Utf8()),
      time: arrow.makeVector({
        type: new Timestamp('America/New_York'),
        data: new BigInt64Array([0n, first, first + 2n, first + 4n, 0n]),
        nullBitmap: new Uint8Array([0b01111]),
        nullCount: 1
      })
    }).slice(1);
    const original = [first, first + 2n, first + 4n, null];
    t.ok(source.getChild('time').data[0].offset > 0, 'fixture uses a nonzero Arrow offset');
    t.deepEqual(rawValues(source.getChild('time')), original, 'fixture keeps raw timestamp units');
    const table = new KeplerTable({info: {id: 'raw-timestamps'}, color: [0, 0, 0]}).updateSchema(
      processArrowBatches(source.batches)
    );
    const container = table.dataContainer;
    const accessor = table.fields[1].valueAccessor;
    t.ok(
      table.upsertRows(
        [
          ['new-b', 2000],
          ['a', 1000],
          ['new-a', 3000],
          ['a', 4000],
          ['new-b', 5000]
        ],
        'id'
      ),
      'mixed replacement and insertion succeeds'
    );
    const expected = [
      4000n * unitsPerMillisecond,
      first + 2n,
      first + 4n,
      null,
      5000n * unitsPerMillisecond,
      3000n * unitsPerMillisecond
    ];
    t.equal(table.dataContainer, container, 'upsert retains the container');
    t.equal(accessor({index: 0}), 4000, 'the existing accessor reads the last replacement');
    t.deepEqual(
      Array.from(container.getColumn(0)),
      ['a', 'b', 'c', 'd', 'new-b', 'new-a'],
      'existing positions and first incoming key order survive repeated keys'
    );
    t.deepEqual(
      rawValues(container.getColumn(1)),
      expected,
      'untouched raw timestamps survive upsert'
    );
    const restored = arrow.tableFromIPC(arrow.tableToIPC(container.getTable()));
    t.deepEqual(
      rawValues(restored.getChild('time')),
      expected,
      'IPC preserves raw timestamps and nulls'
    );
    t.equal(
      restored.schema.fields[1].type.timezone,
      'America/New_York',
      'IPC preserves the timezone'
    );
    t.deepEqual(
      rawValues(source.getChild('time')),
      original,
      'upsert does not mutate the source snapshot'
    );

    const snapshot = container.getTable();
    const revision = table.dataRevision;
    t.notOk(
      table.upsertRows(
        [
          ['a', 6000],
          ['invalid', 'not-a-timestamp']
        ],
        'id'
      ),
      'a later invalid timestamp rejects the whole batch'
    );
    t.equal(container.getTable(), snapshot, 'a rejected upsert retains the Arrow snapshot');
    t.equal(table.dataRevision, revision, 'a rejected upsert does not advance the revision');
    t.deepEqual(
      rawValues(container.getColumn(1)),
      expected,
      'a rejected upsert changes no raw values'
    );
  }
  t.end();
});

test('KeplerTable -> update DuckDB cols payload keeps fields and gpuFilter', async t => {
  const first = arrow.tableFromArrays({
    lat: [1, 2],
    lng: [3, 4]
  });
  const table = new KeplerTable({info: {id: 'duckdb-shape'}, color: [0, 0, 0]});
  await table.importData({
    data: {
      fields: [
        {name: 'lat', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'},
        {name: 'lng', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'}
      ],
      rows: [],
      cols: [first.getChildAt(0), first.getChildAt(1)],
      arrowTable: first
    }
  });
  const gpuFilter = table.gpuFilter;
  const names = table.fields.map(field => field.name);

  const next = arrow.tableFromArrays({
    lat: [1, 2, 3],
    lng: [3, 4, 5]
  });
  // DuckDBTable.update calls super.update({cols, rows: [], fields: []})
  await table.update({
    cols: [next.getChildAt(0), next.getChildAt(1)],
    rows: [],
    fields: []
  });

  t.equal(table.length, 3, 'DuckDB-shaped update should grow rows');
  t.deepEqual(
    table.fields.map(field => field.name),
    names,
    'empty fields must not rebuild schema'
  );
  t.equal(table.gpuFilter, gpuFilter, 'DuckDB-shaped update should keep gpuFilter');
  t.end();
});

test('createNewDataEntry -> existing id updates Arrow table in place', async t => {
  const fields = [
    {name: 'lat', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'},
    {name: 'lng', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'}
  ];
  const batch1 = arrow.tableFromArrays({lat: [1], lng: [2]});
  const datasets = await createNewDataEntryMock({
    info: {id: 'arrow-1'},
    data: {
      fields,
      rows: [],
      cols: [batch1.getChildAt(0), batch1.getChildAt(1)]
    }
  });
  const table = datasets['arrow-1'];
  t.equal(table.length, 1, 'first batch should create the table');
  const gpuFilter = table.gpuFilter;
  const revision = table.dataRevision;

  const batch2 = arrow.tableFromArrays({
    lat: [1, 3],
    lng: [2, 4]
  });
  const task = createNewDataEntry(
    {
      info: {id: 'arrow-1'},
      data: {
        fields,
        rows: [],
        cols: [batch2.getChildAt(0), batch2.getChildAt(1)]
      }
    },
    datasets
  );
  t.equal(task.type, 'UPDATE_TABLE_TASK', 'same id should update instead of creating');

  await task.run(
    async (effectorPrime, success, error) => {
      await effectorPrime(success, error);
    },
    value => {
      t.equal(value, table, 'should return the same table instance');
    }
  );

  t.equal(table.length, 2, 'second batch should replace/grow rows');
  t.equal(table.gpuFilter, gpuFilter, 'incremental createNewDataEntry should keep gpuFilter');
  t.ok(table.dataRevision > revision, 'incremental createNewDataEntry should bump dataRevision');
  t.end();
});

test('KeplerTable -> update rebuilds fields when arrow column names change', async t => {
  const first = arrow.tableFromArrays({
    lat: [1],
    lng: [2]
  });
  const table = new KeplerTable({info: {id: 'arrow-schema'}, color: [0, 0, 0]});
  await table.importData({
    data: {
      fields: [
        {name: 'lat', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'},
        {name: 'lng', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'}
      ],
      rows: [],
      cols: [first.getChildAt(0), first.getChildAt(1)],
      arrowTable: first
    }
  });

  const next = arrow.tableFromArrays({
    lat: [1],
    lng: [2],
    name: ['a']
  });
  await table.update({
    fields: [
      {name: 'lat', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'},
      {name: 'lng', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'},
      {name: 'name', type: ALL_FIELD_TYPES.string, analyzerType: 'STRING'}
    ],
    rows: [],
    cols: [next.getChildAt(0), next.getChildAt(1), next.getChildAt(2)],
    arrowTable: next
  });

  t.equal(table.fields.length, 3, 'column schema change should rebuild fields');
  t.equal(table.fields[2].name, 'name');
  t.end();
});

test('KeplerTable -> update replaces row snapshots', async t => {
  const table = new KeplerTable({info: {id: 'rows'}, color: [0, 0, 0]});
  await table.importData({
    data: {
      fields: [
        {name: 'lat', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'},
        {name: 'lng', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'}
      ],
      rows: [
        [1, 2],
        [3, 4]
      ]
    }
  });
  t.equal(table.length, 2, 'starts with 2 rows');
  table.filterRecord = {name: 'stale'};

  await table.update({
    fields: [
      {name: 'lat', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'},
      {name: 'lng', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'}
    ],
    rows: [
      [10, 20],
      [30, 40],
      [50, 60]
    ]
  });
  t.equal(table.length, 3, 'should grow to the new snapshot');
  t.equal(table.fields[0].valueAccessor({index: 0}), 10, 'accessors should read new values');
  t.equal(table.filteredIndex.length, 3, 'filtered index should match new length');
  t.equal(table.filterRecord, undefined, 'row snapshot should drop stale filterRecord');

  const leftoverArrow = arrow.tableFromArrays({lat: [0], lng: [0]});
  await table.update({
    fields: [
      {name: 'lat', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'},
      {name: 'lng', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'}
    ],
    rows: [[7, 8]],
    arrowTable: leftoverArrow
  });
  t.equal(table.length, 1, 'row snapshot should win over leftover arrowTable');
  t.equal(table.fields[0].valueAccessor({index: 0}), 7);

  await table.update({
    fields: [
      {name: 'lat', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'},
      {name: 'lng', type: ALL_FIELD_TYPES.real, analyzerType: 'FLOAT'},
      {name: 'name', type: ALL_FIELD_TYPES.string, analyzerType: 'STRING'}
    ],
    rows: [['a', 'b', 'c']]
  });
  t.equal(table.fields.length, 3, 'schema change should rebuild fields');
  t.equal(table.length, 1, 'schema change should import the new rows');
  t.end();
});