Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

23.5 KB
Raw Blame
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import test from 'tape';
import cloneDeep from 'es-toolkit/compat/cloneDeep';
import SchemaManager, {datasetSchema, VERSIONS} from '@kepler.gl/schemas';
import {DatasetType, DATASET_FORMATS} from '@kepler.gl/constants';
import * as arrow from 'apache-arrow';
import {KeplerTable, joinDatasets} from '@kepler.gl/table';
import {exportToJsonString, exportMapToHTML, validateInputData} from '@kepler.gl/utils';
import {processArrowTable, processKeplerglJSON} from '@kepler.gl/processors';

// fixtures
import {
  savedStateV0,
  expectedInfo0,
  expectedFields0,
  expectedFields1,
  expectedInfo1
} from 'test/fixtures/state-saved-v0';
import {savedStateV1, v0ExpectedInfo, v0ExpectedFields} from 'test/fixtures/state-saved-v1-1';
import {stateSavedV1_2, v1expectedInfo_2, v1expectedFields_2} from 'test/fixtures/state-saved-v1-2';

/* eslint-disable max-statements */
test('#DatasetSchema -> SchemaManager.parseSavedData', t => {
  const dataSaved = cloneDeep(savedStateV0).datasets;
  const parsedValid = SchemaManager.parseSavedData(dataSaved);

  const expectedRows0 = dataSaved[0].data.allData;

  const expectedDataset0 = {
    info: expectedInfo0,
    data: {
      fields: expectedFields0,
      rows: expectedRows0
    }
  };

  const expectedRows1 = dataSaved[1].data.allData;

  const expectedDataset1 = {
    info: expectedInfo1,
    data: {
      fields: expectedFields1,
      rows: expectedRows1
    }
  };

  t.equal(parsedValid.length, 2, 'should have 2 datasets');
  t.deepEqual(parsedValid[0], expectedDataset0, 'should parse dataset correctly');
  t.deepEqual(parsedValid[0].info, expectedInfo0, 'should parse info correctly');
  t.deepEqual(parsedValid[0].data.fields, expectedFields0, 'should parse fields correctly');
  t.deepEqual(parsedValid[0].data.rows, expectedRows0, 'should parse rows correctly');

  t.deepEqual(parsedValid[1], expectedDataset1, 'should parse dataset correctly');
  t.deepEqual(parsedValid[1].info, expectedInfo1, 'should parse info correctly');
  t.deepEqual(parsedValid[1].data.fields, expectedFields1, 'should parse fields correctly');
  t.deepEqual(parsedValid[1].data.rows, expectedRows1, 'should parse rows correctly');

  t.end();
});
/* eslint-enable max-statements */

test('#DatasetSchema -> SchemaManager.parseSavedData.v1', t => {
  const dataSaved = cloneDeep(savedStateV1).datasets;
  const parsedValid = SchemaManager.parseSavedData(dataSaved);

  const expectedRows = dataSaved[0].data.allData;

  const expectedDataset = {
    info: v0ExpectedInfo,
    data: {
      fields: v0ExpectedFields,
      rows: expectedRows
    }
  };

  t.equal(parsedValid.length, 1, 'should have 1 dataset');

  t.deepEqual(parsedValid[0], expectedDataset, 'should parse dataset correctly');
  t.deepEqual(parsedValid[0].info, v0ExpectedInfo, 'should parse info correctly');
  t.deepEqual(parsedValid[0].data.fields, v0ExpectedFields, 'should parse fields correctly');
  t.deepEqual(parsedValid[0].data.rows, expectedRows, 'should parse rows correctly');

  t.end();
});

test('#DatasetSchema -> SchemaManager.parseSavedData.v1 with ts', t => {
  const dataSaved = cloneDeep(stateSavedV1_2).datasets;
  const parsedValid = SchemaManager.parseSavedData(dataSaved);

  const expectedRows = dataSaved[0].data.allData;
  const expectedDataset = {
    info: v1expectedInfo_2,
    data: {
      fields: v1expectedFields_2,
      rows: expectedRows
    }
  };

  t.equal(parsedValid.length, 1, 'should have 1 dataset');

  t.deepEqual(parsedValid[0], expectedDataset, 'should parse dataset correctly');

  t.deepEqual(parsedValid[0].info, v1expectedInfo_2, 'should parse info correctly');

  t.equal(
    parsedValid[0].data.fields.length,
    v1expectedFields_2.length,
    'should have same number of fields'
  );

  parsedValid[0].data.fields.forEach((actualField, i) => {
    t.deepEqual(
      actualField,
      v1expectedFields_2[i],
      `fields ${actualField.name} should be the same`
    );
  });

  t.deepEqual(parsedValid[0].data.rows, expectedRows, 'should parse rows correctly');

  t.end();
});

test('#DatasetSchema -> save externally-hosted dataset without inlining rows', t => {
  const dataset = {
    id: 'remote-1',
    label: 'quakes.csv',
    color: [1, 2, 3],
    type: DatasetType.EXTERNALLY_HOSTED,
    fields: [{name: 'lat', type: 'real', format: '', analyzerType: 'FLOAT'}],
    metadata: {
      source: 'https://example.com/quakes.csv',
      sourceFormat: 'csv',
      extra: 'should-not-save'
    }
  };

  const saved = datasetSchema[VERSIONS.v1].save(dataset);

  t.deepEqual(saved.allData, [], 'should not inline remote rows');
  t.equal(saved.type, DatasetType.EXTERNALLY_HOSTED, 'should persist dataset type');
  t.deepEqual(
    saved.metadata,
    {source: 'https://example.com/quakes.csv', format: 'csv'},
    'should persist source URL without extra metadata'
  );

  const loaded = SchemaManager.parseSavedData([{version: 'v1', data: saved}]);
  t.equal(loaded.length, 1, 'should parse one dataset');
  t.equal(loaded[0].info.type, DatasetType.EXTERNALLY_HOSTED, 'should restore type');
  t.equal(loaded[0].info.id, 'remote-1', 'should restore id');
  t.equal(loaded[0].metadata.source, 'https://example.com/quakes.csv', 'should restore source');
  t.equal(loaded[0].metadata.sourceFormat, 'csv', 'should map saved format to sourceFormat');
  t.equal(loaded[0].metadata.format, undefined, 'should not collide with DATASET_FORMATS');
  t.deepEqual(loaded[0].data.rows, [], 'loaded rows should stay empty until refetch');

  t.end();
});

test('#DatasetSchema -> Zarr saves its source without embedding metadata or coordinate rows', t => {
  const values = Array.from({length: 20_000}, (_, index) => index);
  const dimension = {name: 'time', size: values.length, values};
  const source = {
    url: 'https://example.com/forecast.zarr',
    variable: 'temperature',
    label: 'Forecast',
    attribution: 'Weather provider'
  };
  const metadata = {
    ...source,
    variables: [{path: 'temperature', nonSpatialDims: [dimension]}],
    nonSpatialDims: [dimension],
    timeDimension: dimension,
    levels: [{path: '.', shape: [2048, 2048], chunks: [128, 128]}]
  };
  const dataset = {
    id: 'forecast',
    label: 'My forecast',
    type: DatasetType.ZARR,
    color: [1, 2, 3],
    disableDataOperation: true,
    metadata,
    fields: [{name: 'time', type: 'timestamp'}],
    dataContainer: {
      flattenData() {
        t.fail('must not materialize coordinate rows during export');
        return values.map(value => [value]);
      }
    }
  };

  const saved = datasetSchema[VERSIONS.v1].save(dataset);
  t.deepEqual(saved.metadata, source, 'saves only the source, default variable and attribution');
  t.deepEqual(saved.allData, [], 'does not save coordinate rows');
  t.deepEqual(saved.fields, [], 'does not save synthetic coordinate fields');
  t.equal(saved.disableDataOperation, true, 'preserves dataset behavior');
  t.ok(JSON.stringify(saved).length < 1000, 'export size is independent of large coordinate lists');
  t.equal(dataset.metadata, metadata, 'does not replace live metadata');
  t.equal(dataset.metadata.timeDimension.values, values, 'keeps live coordinates available');

  const [loaded] = SchemaManager.parseSavedData([{version: 'v1', data: saved}]);
  t.deepEqual(loaded.metadata, source, 'retains what the metadata refresh needs');
  t.equal(loaded.info.id, dataset.id, 'preserves dataset identity');
  t.equal(loaded.info.label, dataset.label, 'preserves the user label');
  t.equal(loaded.info.type, DatasetType.ZARR, 'restores the remote dataset type');

  const root = datasetSchema[VERSIONS.v1].save({...dataset, metadata: {...metadata, variable: ''}});
  t.equal(root.metadata.variable, '', 'retains a root-array variable path');
  const automatic = datasetSchema[VERSIONS.v1].save({...dataset, metadata: {url: source.url}});
  t.deepEqual(automatic.metadata, {url: source.url}, 'allows restoring the default variable');

  const legacy = SchemaManager.parseSavedData([
    {
      version: 'v1',
      data: {...saved, metadata}
    }
  ])[0];
  t.deepEqual(legacy.metadata, metadata, 'older maps with embedded metadata still load');
  t.end();
});

test('#DatasetSchema -> save externally-hosted dataset refresh interval', t => {
  const dataset = {
    id: 'remote-1',
    label: 'quakes.csv',
    color: [1, 2, 3],
    type: DatasetType.EXTERNALLY_HOSTED,
    fields: [{name: 'lat', type: 'real', format: '', analyzerType: 'FLOAT'}],
    metadata: {
      source: 'https://example.com/quakes.csv',
      sourceFormat: 'csv',
      refreshIntervalMs: 60_000,
      etag: '"skip-me"',
      refreshStatus: 'loading'
    }
  };

  const saved = datasetSchema[VERSIONS.v1].save(dataset);
  t.deepEqual(
    saved.metadata,
    {source: 'https://example.com/quakes.csv', format: 'csv', refreshIntervalMs: 60_000},
    'should persist refresh interval without runtime status'
  );

  const loaded = SchemaManager.parseSavedData([{version: 'v1', data: saved}]);
  t.equal(loaded[0].metadata.refreshIntervalMs, 60_000, 'should restore refresh interval');
  t.equal(loaded[0].metadata.etag, undefined, 'should not restore etag');

  t.end();
});

test('#DatasetSchema -> save externally-hosted dataset custom refresh interval', t => {
  const dataset = {
    id: 'remote-1',
    label: 'quakes.csv',
    color: [1, 2, 3],
    type: DatasetType.EXTERNALLY_HOSTED,
    fields: [{name: 'lat', type: 'real', format: '', analyzerType: 'FLOAT'}],
    metadata: {
      source: 'https://example.com/quakes.csv',
      sourceFormat: 'csv',
      refreshIntervalMs: 45_000
    }
  };

  const saved = datasetSchema[VERSIONS.v1].save(dataset);
  t.equal(saved.metadata.refreshIntervalMs, 45_000, 'should persist a custom interval');

  const loaded = SchemaManager.parseSavedData([{version: 'v1', data: saved}]);
  t.equal(loaded[0].metadata.refreshIntervalMs, 45_000, 'should restore a custom interval');

  t.end();
});

function arrowDataset(columns) {
  const table = new arrow.Table(columns);
  const data = processArrowTable({shape: 'arrow-table', data: table});
  return new KeplerTable({
    color: [1, 2, 3],
    metadata: {format: DATASET_FORMATS.arrow}
  }).updateSchema(data);
}

function reloadDataset(saved) {
  const map = {datasets: [{version: 'v1', data: saved}], config: {version: 'v1', config: {}}};
  const parsed = processKeplerglJSON(JSON.parse(exportToJsonString(map)));
  return new KeplerTable({color: [1, 2, 3]}).updateSchema(
    validateInputData(parsed.datasets[0].data)
  );
}

test('#DatasetSchema -> Int64 values survive JSON, HTML and reload without precision loss', t => {
  const dataset = arrowDataset({
    signed: arrow.vectorFromArray(
      [42n, 0n, 9007199254740991n, -9223372036854775808n, null],
      new arrow.Int64()
    ),
    unsigned: arrow.vectorFromArray(
      [42n, 0n, 9007199254740992n, 18446744073709551615n, null],
      new arrow.Uint64()
    )
  });
  const originalRows = dataset.dataContainer.flattenData();
  const saved = datasetSchema[VERSIONS.v1].save(dataset);
  const expected = [
    [42, 42],
    [0, 0],
    [9007199254740991, '9007199254740992'],
    ['-9223372036854775808', '18446744073709551615'],
    [null, null]
  ];
  t.deepEqual(
    saved.allData,
    expected,
    'uses numbers in the safe range and exact decimal strings outside it'
  );
  t.doesNotThrow(() => {
    const reloaded = reloadDataset(saved);
    t.deepEqual(reloaded.dataContainer.flattenData(), expected, 'JSON reload retains exact values');
    t.deepEqual(
      datasetSchema[VERSIONS.v1].save(reloaded).allData,
      expected,
      'resaving retains exact values'
    );
  }, 'JSON export is valid and reloads');
  t.doesNotThrow(() => {
    const html = exportMapToHTML({
      datasets: [{version: 'v1', data: saved}],
      config: {version: 'v1', config: {}}
    });
    const embedded = JSON.parse(html.match(/const datasets = (.*);/)[1]);
    t.deepEqual(
      reloadDataset(embedded[0].data).dataContainer.flattenData(),
      expected,
      'HTML embedded data reloads exactly'
    );
  }, 'HTML export succeeds');
  t.deepEqual(
    dataset.dataContainer.flattenData(),
    originalRows,
    'saving does not mutate Arrow values'
  );
  t.end();
});

test('#DatasetSchema -> nested Arrow Int64 values survive a complete JSON backup and reload', t => {
  const item = new arrow.Struct([
    new arrow.Field('signed', new arrow.Int64(), true),
    new arrow.Field('unsigned', new arrow.Uint64(), true)
  ]);
  const type = new arrow.Struct([
    new arrow.Field('items', new arrow.List(new arrow.Field('item', item, true)), true)
  ]);
  const values = [
    {
      items: [
        {signed: 42n, unsigned: 9007199254740991n},
        {signed: -9223372036854775808n, unsigned: 18446744073709551615n},
        {signed: null, unsigned: 0n},
        null
      ]
    },
    {items: []},
    {items: null},
    null
  ];
  const source = new arrow.Table({nested: arrow.vectorFromArray(values, type)});
  const ipc = arrow.tableFromIPC(arrow.tableToIPC(source, 'file'));
  const dataset = new KeplerTable({info: {id: 'nested'}, color: [1, 2, 3]}).updateSchema(
    processArrowTable({shape: 'arrow-table', data: ipc})
  );
  const expected = [
    [
      {
        items: [
          {signed: 42, unsigned: 9007199254740991},
          {signed: '-9223372036854775808', unsigned: '18446744073709551615'},
          {signed: null, unsigned: 0},
          null
        ]
      }
    ],
    [{items: []}],
    [{items: null}],
    [null]
  ];
  const saved = datasetSchema[VERSIONS.v1].save(dataset);
  t.doesNotThrow(() => {
    const payload = JSON.parse(
      exportToJsonString({
        datasets: [{version: 'v1', data: saved}],
        config: {version: 'v1', config: {}}
      })
    );
    t.deepEqual(payload.datasets[0].data.allData, expected, 'full backup has exact nested values');
    const reloaded = reloadDataset(payload.datasets[0].data);
    t.deepEqual(reloaded.dataContainer.flattenData(), expected, 'reload retains nesting and nulls');
    t.deepEqual(datasetSchema[VERSIONS.v1].save(reloaded).allData, expected, 'resave is stable');
    const html = exportMapToHTML(payload);
    t.deepEqual(
      JSON.parse(html.match(/const datasets = (.*);/)[1]),
      payload.datasets,
      'HTML uses the same saved values'
    );
  }, 'backup is valid JSON and reloads');
  t.equal(
    dataset.dataContainer.valueAt(0, 0).items.get(1).signed,
    -9223372036854775808n,
    'live Arrow values are not mutated'
  );
  t.equal(
    ipc.getChild('nested').get(0).items.get(1).unsigned,
    18446744073709551615n,
    'source buffers are not mutated'
  );

  const rows = [[{items: [1n, null, {value: 9007199254740993n}]}]];
  const rowTable = new KeplerTable({info: {id: 'nested-rows'}, color: [1, 2, 3]}).updateSchema({
    fields: [{name: 'nested', type: 'object'}],
    rows
  });
  t.deepEqual(
    datasetSchema[VERSIONS.v1].save(rowTable).allData,
    [[{items: [1, null, {value: '9007199254740993'}]}]],
    'row-backed nested values use the same policy'
  );
  t.equal(rows[0][0].items[2].value, 9007199254740993n, 'live row-backed objects are not mutated');
  t.end();
});

test('#DatasetSchema -> nested custom JSON serializers retain their representations', t => {
  class Identifier {
    constructor() {
      this.raw = 9007199254740993n;
    }
    toJSON() {
      return {id: this.raw.toString()};
    }
  }
  class CachedValue {
    constructor() {
      this.cache = {owner: this};
    }
    toJSON() {
      return {value: 'valid'};
    }
  }
  const custom = {
    raw: 42n,
    toJSON() {
      return {value: this.raw.toString()};
    }
  };
  custom.cache = custom;
  const identifier = new Identifier();
  const cached = new CachedValue();
  const rows = [[{identifier, cached, custom, ordinary: {value: 42n}}]];
  const dataset = new KeplerTable({info: {id: 'custom-json'}, color: [1, 2, 3]}).updateSchema({
    fields: [{name: 'nested', type: 'object'}],
    rows
  });

  const saved = datasetSchema[VERSIONS.v1].save(dataset);
  const expected = [
    [
      {
        identifier: {id: '9007199254740993'},
        cached: {value: 'valid'},
        custom: {value: '42'},
        ordinary: {value: 42}
      }
    ]
  ];
  t.deepEqual(
    JSON.parse(exportToJsonString(saved)).allData,
    expected,
    'preserves class and plain-object serializers while normalizing ordinary nested values'
  );
  t.deepEqual(reloadDataset(saved).dataContainer.flattenData(), expected, 'custom values reload');
  t.equal(saved.allData[0][0].identifier, identifier, 'retains the custom class instance');
  t.equal(saved.allData[0][0].cached, cached, 'does not traverse the cyclic internal cache');
  t.equal(identifier.raw, 9007199254740993n, 'does not change live class data');
  t.equal(rows[0][0].ordinary.value, 42n, 'does not change ordinary live nested data');
  t.end();
});

test('#DatasetSchema -> rejects ordinary circular values without a JSON serializer', t => {
  const cycle = {};
  cycle.self = cycle;
  const dataset = new KeplerTable({info: {id: 'circular'}, color: [1, 2, 3]}).updateSchema({
    fields: [{name: 'nested', type: 'object'}],
    rows: [[cycle]]
  });
  t.throws(
    () => datasetSchema[VERSIONS.v1].save(dataset),
    /Cannot export circular dataset values/,
    'still rejects an actual serialization cycle'
  );
  t.end();
});

test('#DatasetSchema -> joined Int64 rows save and reload without changing live rows', t => {
  const values = [
    0n,
    10n,
    20n,
    -9007199254740991n,
    9007199254740991n,
    -9223372036854775808n,
    9007199254740992n,
    9007199254740993n,
    null
  ];
  const source = arrowDataset({
    key: arrow.vectorFromArray(values.map((_, index) => String(index))),
    signed: arrow.vectorFromArray(values, new arrow.Int64()),
    unsigned: arrow.vectorFromArray(
      values.map(value => (value === null ? null : value < 0n ? 18446744073709551615n : value)),
      new arrow.Uint64()
    )
  });
  const proto = joinDatasets(source, source, {
    leftField: 'key',
    rightField: 'key',
    type: 'LEFT',
    rightColumns: []
  });
  const original = proto.data.rows.map(row => [...row]);
  const expected = [
    ['0', 0, 0],
    ['1', 10, 10],
    ['2', 20, 20],
    ['3', -9007199254740991, '18446744073709551615'],
    ['4', 9007199254740991, 9007199254740991],
    ['5', '-9223372036854775808', '18446744073709551615'],
    ['6', '9007199254740992', '9007199254740992'],
    ['7', '9007199254740993', '9007199254740993'],
    ['8', null, null]
  ];
  const dataset = new KeplerTable({
    info: proto.info,
    metadata: proto.metadata,
    color: [0, 0, 0]
  }).updateSchema(proto.data);
  const saved = datasetSchema[VERSIONS.v1].save(dataset);
  t.deepEqual(saved.allData, expected, 'safe numbers and exact unsafe decimal strings');
  t.equal(
    reloadDataset(saved).getValue('signed', 7),
    '9007199254740993',
    'in-memory saved schema also omits the live BigInt representation'
  );
  t.deepEqual(dataset.dataContainer.flattenData(), original, 'live rows are not mutated');
  t.deepEqual(source.dataContainer.flattenData(), original, 'source Arrow values are not mutated');
  t.doesNotThrow(() => {
    const json = exportToJsonString({
      datasets: [{version: 'v1', data: saved}],
      config: {version: 'v1', config: {}}
    });
    const parsed = processKeplerglJSON(JSON.parse(json)).datasets[0];
    const reloaded = new KeplerTable({
      info: parsed.info,
      metadata: parsed.metadata,
      color: [0, 0, 0]
    }).updateSchema(validateInputData(parsed.data));
    t.deepEqual(reloaded.dataContainer.flattenData(), expected, 'JSON reload retains exact cells');
    t.equal(
      reloaded.fields[1].valueAccessor({index: 7}),
      '9007199254740993',
      'accepted unsafe-string numeric limitation unchanged'
    );
    t.deepEqual(
      datasetSchema[VERSIONS.v1].save(reloaded).allData,
      expected,
      'resave retains exact cells'
    );
    const html = exportMapToHTML({
      datasets: [{version: 'v1', data: saved}],
      config: {version: 'v1', config: {}}
    });
    const embedded = JSON.parse(html.match(/const datasets = (.*);/)[1]);
    t.deepEqual(embedded[0].data.allData, expected, 'HTML retains the same exact cells');
    t.deepEqual(
      reloadDataset(embedded[0].data).dataContainer.flattenData(),
      expected,
      'HTML reload retains exact cells'
    );
  }, 'JSON and HTML exports are valid');
  t.end();
});

test('#DatasetSchema -> string-typed Arrow Int64 joins still export exact values', t => {
  const source = new KeplerTable({info: {id: 'string-arrow'}, color: [0, 0, 0]}).updateSchema({
    fields: [
      {name: 'key', type: 'string'},
      {name: 'value', type: 'string'}
    ],
    rows: [],
    cols: [
      arrow.vectorFromArray(['a', 'b']),
      arrow.vectorFromArray([10n, 9007199254740993n], new arrow.Int64())
    ]
  });
  const proto = joinDatasets(source, source, {
    leftField: 'key',
    rightField: 'key',
    type: 'LEFT',
    rightColumns: []
  });
  const table = new KeplerTable({info: proto.info, color: [0, 0, 0]}).updateSchema(proto.data);
  t.deepEqual(datasetSchema.v1.save(table).allData, [
    ['a', 10],
    ['b', '9007199254740993']
  ]);
  t.equal(table.getValue('value', 1), 9007199254740993n);
  t.end();
});

test('#DatasetSchema -> row numeric controls and empty rows remain unchanged', t => {
  for (const rows of [[], [[null], [null]], [[10], [20], [null]], [[10], null, [], [20]]]) {
    const dataset = new KeplerTable({
      color: [0, 0, 0]
    }).updateSchema({fields: [{name: 'value', type: 'integer', analyzerType: 'INT'}], rows});
    const saved = datasetSchema[VERSIONS.v1].save(dataset);
    t.deepEqual(JSON.parse(exportToJsonString(saved)).allData, rows, 'control saves unchanged');
    t.deepEqual(dataset.dataContainer.flattenData(), rows, 'control source unchanged');
    t.equal(saved.allData, rows, 'no row copy is needed when there are no BigInts');
  }
  t.end();
});

test('#DatasetSchema -> BigInt normalization copies only affected rows and preserves other cells', t => {
  const date = new Date('2020-01-01T00:00:00Z');
  const rows = [null, [10n, date, '9007199254740993'], [20, null, 'unchanged'], null];
  const dataset = new KeplerTable({color: [0, 0, 0]}).updateSchema({
    fields: [
      {name: 'value', type: 'integer'},
      {name: 'date', type: 'date'},
      {name: 'text', type: 'string'}
    ],
    rows
  });
  const saved = datasetSchema[VERSIONS.v1].save(dataset);
  t.notEqual(saved.allData, rows, 'the outer array is copied for a conversion');
  t.deepEqual(
    saved.allData,
    [null, [10, date, '9007199254740993'], rows[2], null],
    'only BigInt cells change'
  );
  t.notEqual(saved.allData[1], rows[1], 'the affected row is copied');
  t.equal(saved.allData[2], rows[2], 'the unchanged row is reused');
  t.equal(saved.allData[1][1], date, 'row dates are not converted by Arrow temporal handling');
  t.equal(dataset.dataContainer.valueAt(1, 0), 10n, 'live storage remains exact');
  t.end();
});

test('#DatasetSchema -> null Arrow timestamps remain null through save and reload', t => {
  const dataset = arrowDataset({
    time: arrow.vectorFromArray([null, 0, 1700000000123], new arrow.TimestampMillisecond())
  });
  const saved = datasetSchema[VERSIONS.v1].save(dataset);
  const expected = [[null], ['1970-01-01T00:00:00.000Z'], ['2023-11-14T22:13:20.123Z']];
  t.deepEqual(saved.allData, expected, 'distinguishes missing timestamps from valid epoch zero');
  const reloaded = reloadDataset(saved);
  t.deepEqual(
    reloaded.dataContainer.flattenData(),
    expected,
    'JSON reload preserves missing and nonmissing values'
  );
  t.deepEqual(
    datasetSchema[VERSIONS.v1].save(reloaded).allData,
    expected,
    'resaving preserves nulls'
  );
  t.deepEqual(
    dataset.dataContainer.flattenData(),
    [[null], [0], [1700000000123]],
    'saving leaves source values unchanged'
  );
  t.end();
});