Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

feat(data-table): show filtered rows and column stats (#3815)

Igor Dykhtadfb3bcfd9a4f
diff --git a/src/components/src/modal-container.tsx b/src/components/src/modal-container.tsx
index e03fb82ff..6ae4ebe8d 100644
--- a/src/components/src/modal-container.tsx
+++ b/src/components/src/modal-container.tsx
@@ -455,6 +455,7 @@ export default function ModalContainerFactory(
}
setColumnDisplayFormat={visStateActions.setColumnDisplayFormat}
loadColumnStats={visStateActions.loadColumnStats}
+ applyCPUFilter={visStateActions.applyCPUFilter}
uiStateActions={uiStateActions}
uiState={uiState}
/>
diff --git a/src/components/src/modals/data-table-modal-column-edit.spec.tsx b/src/components/src/modals/data-table-modal-column-edit.spec.tsx
index 0de89921e..4e21bd00b 100644
--- a/src/components/src/modals/data-table-modal-column-edit.spec.tsx
+++ b/src/components/src/modals/data-table-modal-column-edit.spec.tsx
@@ -75,6 +75,13 @@ function renderModal(type: string, hiddenColumns: string[] = []) {
}
describe('data table column edits', () => {
+ test('does not offer filtered data when CPU filtering is unavailable', () => {
+ const {container} = renderModal('local');
+
+ expect(container.querySelector('.data-view-filtered')).toBeNull();
+ expect(container.querySelector('.data-view-all')?.getAttribute('aria-pressed')).toBe('true');
+ });
+
test('create filter opens the filter panel and closes the table', () => {
const {createFilterFromColumn, toggleSidePanel, toggleModal} = renderModal('local');
diff --git a/src/components/src/modals/data-table-modal.tsx b/src/components/src/modals/data-table-modal.tsx
index 9ae8fe29e..cc8fd75db 100644
--- a/src/components/src/modals/data-table-modal.tsx
+++ b/src/components/src/modals/data-table-modal.tsx
@@ -13,11 +13,20 @@ import KeplerTable, {
isTabularDatasetForOps
} from '@kepler.gl/table';
import {UIStateActions} from '@kepler.gl/actions';
+import {FormattedMessage} from '@kepler.gl/localization';
import {UiState} from '@kepler.gl/types';
-import {getApplicationConfig} from '@kepler.gl/utils';
+import {
+ collectColumnValues,
+ ColumnStats,
+ computeColumnStats,
+ createIndexedDataContainer,
+ DataContainerInterface,
+ getApplicationConfig
+} from '@kepler.gl/utils';
import {Gear} from '../common/icons';
import Portaled from '../common/portaled';
import DataTableConfigFactory from '../common/data-table/display-format';
+import {SelectionButton} from '../common/styled-components';
import {BaseComponentProps} from '../types';
import {STATS_WIDTH} from '../common/data-table/column-statistics-components';
@@ -93,6 +102,107 @@ const StyledConfigureButton = styled.div`
cursor: pointer;
`;
+const DataViewBar = styled.div.attrs({
+ className: 'data-view-controls'
+})`
+ align-items: center;
+ border-bottom: 1px solid ${props => props.theme.dropdownListBorderTopLT || '#D3D8E0'};
+ display: flex;
+ flex: 0 0 auto;
+ padding: 8px 16px 12px;
+`;
+
+const DataViewLabel = styled.div.attrs({
+ className: 'data-view-label'
+})`
+ color: ${props => props.theme.textColorLT};
+ font-size: 14px;
+ font-weight: 600;
+ margin-right: 12px;
+`;
+
+const DataViewButton = styled(SelectionButton)`
+ color: ${props => (props.selected ? props.theme.selectionBtnActColor : props.theme.textColorLT)};
+ font-size: 12px;
+ font-weight: 500;
+`;
+
+const DataViewRowCount = styled.div.attrs({
+ className: 'data-view-row-count'
+})`
+ color: ${props => props.theme.subtextColorLT};
+ font-size: 12px;
+ margin-left: 6px;
+`;
+
+/**
+ * Row indexes to display for the filtered view, in the active table sort order.
+ * Indexes stay in the parent dataset so an indexed container can map them.
+ */
+function filteredDisplayIndexes(
+ filteredIdxCPU: number[] | null | undefined,
+ sortOrder: number[] | null | undefined
+): number[] {
+ if (!filteredIdxCPU) {
+ return [];
+ }
+ if (!sortOrder?.length) {
+ return filteredIdxCPU;
+ }
+ const filtered = new Set(filteredIdxCPU);
+ return sortOrder.filter(index => filtered.has(index));
+}
+
+function filteredColumnValues(
+ field: {fieldIdx?: number; filterProps?: object | null},
+ dataContainer: DataContainerInterface,
+ filteredIdxCPU: number[]
+): unknown[] {
+ const mappedValue =
+ field.filterProps &&
+ 'mappedValue' in field.filterProps &&
+ Array.isArray(field.filterProps.mappedValue)
+ ? field.filterProps.mappedValue
+ : undefined;
+ if (mappedValue && mappedValue.length === dataContainer.numRows()) {
+ return filteredIdxCPU.map(index => mappedValue[index]);
+ }
+ return collectColumnValues(
+ createIndexedDataContainer(dataContainer, filteredIdxCPU),
+ field.fieldIdx ?? 0
+ );
+}
+
+type FilteredColumnStatsState = {
+ key: number[] | undefined;
+ stats: Record<string, ColumnStats>;
+ loading: Record<string, boolean>;
+};
+
+const EMPTY_FILTERED_COLUMN_STATS: FilteredColumnStatsState = {
+ key: undefined,
+ stats: {},
+ loading: {}
+};
+
+function tableDataForView(
+ dataContainer: DataContainerInterface,
+ filteredIdxCPU: number[] | null | undefined,
+ sortOrder: number[] | null | undefined,
+ showFiltered: boolean
+): {dataContainer: DataContainerInterface; sortOrder: number[] | null | undefined} {
+ if (!showFiltered) {
+ return {dataContainer, sortOrder};
+ }
+ return {
+ dataContainer: createIndexedDataContainer(
+ dataContainer,
+ filteredDisplayIndexes(filteredIdxCPU, sortOrder)
+ ),
+ sortOrder: null
+ };
+}
+
interface DatasetTabsUnmemoizedProps {
activeDataset: KeplerTable;
datasets: Datasets;
@@ -149,6 +259,7 @@ interface DataTableModalProps {
datasets: Datasets;
showDatasetTable: (id: string) => void;
showTab?: boolean;
+ applyCPUFilter?: (dataId: string | string[]) => void;
setColumnDisplayFormat: (
dataId: string,
formats: {
@@ -180,11 +291,21 @@ function DataTableModalFactory(
showDatasetTable,
uiStateActions,
showTab = true,
+ applyCPUFilter,
setColumnDisplayFormat: setColumnDisplayFormatProp,
loadColumnStats
}) => {
const [showConfig, setShowConfig] = useState(false);
+ const [showFiltered, setShowFiltered] = useState(false);
const datasetCellSizeCache = useRef<Record<string, any>>({});
+ // Dataset revision covered by the latest CPU filter request.
+ const lastFilteredDataset = useRef<{
+ id: string;
+ filterRecord: unknown;
+ dataRevision: number | undefined;
+ dataContainer: DataContainerInterface | undefined;
+ fields: unknown;
+ } | null>(null);
const enableColumnStats = getApplicationConfig().enableColumnStats;
@@ -202,29 +323,11 @@ function DataTableModalFactory(
return (fields?.map(field => field.name) || []).filter(name => !hidden.has(name));
}, [fields, hiddenColumns]);
- const colMeta = useMemo(
- () =>
- fields?.reduce(
- (
- acc,
- {name, displayName, type, filterProps, format, displayFormat, isLoadingStats},
- colIdx
- ) => ({
- ...acc,
- [name]: {
- name: displayName || name,
- colIdx,
- type,
- ...(format ? {format} : {}),
- ...(displayFormat ? {displayFormat} : {}),
- ...(filterProps?.columnStats ? {columnStats: filterProps.columnStats} : {}),
- ...(typeof isLoadingStats === 'boolean' ? {isLoadingStats} : {})
- }
- }),
- {}
- ) || {},
- [fields]
+ const [filteredColumnStats, setFilteredColumnStats] = useState<FilteredColumnStatsState>(
+ EMPTY_FILTERED_COLUMN_STATS
);
+ const loadingColumnNames = useRef<Set<string>>(new Set());
+ const filteredStatsKey = useRef<number[] | undefined>(undefined);
const [cellSizeCache, setCellSizeCache] = useState<
Record<string, {row: number; header: number}>
@@ -349,11 +452,206 @@ function DataTableModalFactory(
setShowConfig(false);
}, []);
- if (!datasets || !dataId) {
- return null;
+ const requestFilteredData = useCallback(
+ (id: string) => {
+ if (!id || !applyCPUFilter) {
+ return;
+ }
+ const dataset = datasets?.[id];
+ lastFilteredDataset.current = {
+ id,
+ filterRecord: dataset?.filterRecord,
+ dataRevision: dataset?.dataRevision,
+ dataContainer: dataset?.dataContainer,
+ fields: dataset?.fields
+ };
+ applyCPUFilter(id);
+ },
+ [applyCPUFilter, datasets]
+ );
+
+ const onSelectAllData = useCallback(() => {
+ setShowFiltered(false);
+ }, []);
+
+ const onSelectFilteredData = useCallback(() => {
+ if (!dataId || !applyCPUFilter) {
+ return;
+ }
+ requestFilteredData(dataId);
+ setShowFiltered(true);
+ }, [applyCPUFilter, dataId, requestFilteredData]);
+
+ const onShowDatasetTable = useCallback(
+ (id: string) => {
+ if (showFiltered) {
+ requestFilteredData(id);
+ }
+ if (showDatasetTable) {
+ showDatasetTable(id);
+ }
+ },
+ [requestFilteredData, showDatasetTable, showFiltered]
+ );
+
+ const activeDataset = datasets && dataId ? datasets[dataId] : undefined;
+ const sourceContainer = activeDataset?.dataContainer;
+ const filteredIdxCPU = activeDataset?.filteredIdxCPU;
+ const sortOrder = activeDataset?.sortOrder;
+
+ // Refresh filtered indexes when the active dataset, its filters, or its data changes.
+ useEffect(() => {
+ if (!showFiltered || !dataId || !activeDataset) {
+ return;
+ }
+ const lastDataset = lastFilteredDataset.current;
+ if (
+ lastDataset?.id === dataId &&
+ lastDataset.filterRecord === activeDataset.filterRecord &&
+ lastDataset.dataRevision === activeDataset.dataRevision &&
+ lastDataset.dataContainer === sourceContainer &&
+ lastDataset.fields === activeDataset.fields
+ ) {
+ return;
+ }
+ requestFilteredData(dataId);
+ }, [activeDataset, dataId, requestFilteredData, showFiltered, sourceContainer]);
+
+ const displayData = useMemo(
+ () =>
+ sourceContainer
+ ? tableDataForView(sourceContainer, filteredIdxCPU, sortOrder, showFiltered)
+ : null,
+ [filteredIdxCPU, showFiltered, sortOrder, sourceContainer]
+ );
+
+ const filteredStatsKeyValue =
+ showFiltered && Array.isArray(filteredIdxCPU) ? filteredIdxCPU : undefined;
+ if (filteredStatsKey.current !== filteredStatsKeyValue) {
+ filteredStatsKey.current = filteredStatsKeyValue;
+ loadingColumnNames.current = new Set();
+ }
+
+ const visibleFilteredStats =
+ filteredColumnStats.key === filteredStatsKeyValue
+ ? filteredColumnStats.stats
+ : EMPTY_FILTERED_COLUMN_STATS.stats;
+ const visibleFilteredLoading =
+ filteredColumnStats.key === filteredStatsKeyValue
+ ? filteredColumnStats.loading
+ : EMPTY_FILTERED_COLUMN_STATS.loading;
+
+ const handleLoadColumnStats = useCallback(
+ (id: string, fieldName: string | string[]) => {
+ const indexes = filteredStatsKey.current;
+ if (!showFiltered || !indexes || !sourceContainer) {
+ loadColumnStats?.(id, fieldName);
+ return;
+ }
+
+ const names = (Array.isArray(fieldName) ? fieldName : [fieldName]).filter(
+ name => !visibleFilteredStats[name] && !loadingColumnNames.current.has(name)
+ );
+ if (!names.length || !fields) {
+ return;
+ }
+
+ names.forEach(name => loadingColumnNames.current.add(name));
+ setFilteredColumnStats(current => {
+ const base = current.key === indexes ? current : {key: indexes, stats: {}, loading: {}};
+ const loading = {...base.loading};
+ names.forEach(name => {
+ loading[name] = true;
+ });
+ return {...base, loading};
+ });
+
+ const container = sourceContainer;
+ const fieldsToScan = fields;
+ void (async () => {
+ await new Promise(resolve => {
+ setTimeout(resolve, 0);
+ });
+ if (filteredStatsKey.current !== indexes) {
+ return;
+ }
+
+ const computed: Record<string, ColumnStats> = {};
+ for (const name of names) {
+ const field = fieldsToScan.find(candidate => candidate.name === name);
+ if (!field) {
+ continue;
+ }
+ computed[name] = computeColumnStats(
+ filteredColumnValues(field, container, indexes),
+ field.type
+ );
+ if (filteredStatsKey.current !== indexes) {
+ return;
+ }
+ }
+
+ setFilteredColumnStats(current => {
+ if (filteredStatsKey.current !== indexes) {
+ return current;
+ }
+ const base = current.key === indexes ? current : {key: indexes, stats: {}, loading: {}};
+ const loading = {...base.loading};
+ names.forEach(name => {
+ loadingColumnNames.current.delete(name);
+ delete loading[name];
+ });
+ return {key: indexes, stats: {...base.stats, ...computed}, loading};
+ });
+ })();
+ },
+ [fields, loadColumnStats, showFiltered, sourceContainer, visibleFilteredStats]
+ );
+
+ const colMeta = useMemo(
+ () =>
+ fields?.reduce(
+ (
+ acc,
+ {name, displayName, type, filterProps, format, displayFormat, isLoadingStats},
+ colIdx
+ ) => {
+ const waitingForFilter = showFiltered && !Array.isArray(filteredIdxCPU);
+ const stats = showFiltered ? visibleFilteredStats[name] : filterProps?.columnStats;
+ const loading = showFiltered
+ ? waitingForFilter || Boolean(visibleFilteredLoading[name])
+ : isLoadingStats;
+
+ return {
+ ...acc,
+ [name]: {
+ name: displayName || name,
+ colIdx,
+ type,
+ ...(format ? {format} : {}),
+ ...(displayFormat ? {displayFormat} : {}),
+ ...(stats ? {columnStats: stats} : {}),
+ ...(typeof loading === 'boolean' ? {isLoadingStats: loading} : {})
+ }
+ };
+ },
+ {}
+ ) || {},
+ [fields, filteredIdxCPU, showFiltered, visibleFilteredLoading, visibleFilteredStats]
+ );
+
+ let rowCount: number | undefined;
+ if (!sourceContainer || (showFiltered && !filteredIdxCPU)) {
+ rowCount = undefined;
+ } else if (showFiltered) {
+ rowCount = displayData?.dataContainer.numRows();
+ } else {
+ rowCount = sourceContainer.numRows();
}
- const activeDataset = datasets[dataId];
+ if (!datasets || !dataId || !activeDataset || !displayData) {
+ return null;
+ }
const canMutateColumns = canMutateDatasetSchema(activeDataset);
const canCreateFilter = isTabularDatasetForOps(activeDataset);
@@ -365,9 +663,41 @@ function DataTableModalFactory(
<DatasetTabs
activeDataset={activeDataset}
datasets={datasets}
- showDatasetTable={showDatasetTable}
+ showDatasetTable={onShowDatasetTable}
/>
) : null}
+ <DataViewBar>
+ <DataViewLabel>
+ <FormattedMessage id="modal.dataTable.show" />
+ </DataViewLabel>
+ <DataViewButton
+ className="data-view-all"
+ selected={!showFiltered}
+ onClick={onSelectAllData}
+ role="button"
+ aria-pressed={!showFiltered}
+ >
+ <FormattedMessage id="modal.dataTable.allData" />
+ </DataViewButton>
+ {applyCPUFilter ? (
+ <DataViewButton
+ className="data-view-filtered"
+ selected={showFiltered}
+ onClick={onSelectFilteredData}
+ role="button"
+ aria-pressed={showFiltered}
+ >
+ <FormattedMessage id="modal.dataTable.filteredData" />
+ </DataViewButton>
+ ) : null}
+ <DataViewRowCount>
+ {rowCount === undefined ? (
+ '-'
+ ) : (
+ <FormattedMessage id="modal.dataTable.rowCount" values={{rowCount}} />
+ )}
+ </DataViewRowCount>
+ </DataViewBar>
<StyledConfigureButton className="display-config-button">
{hiddenColumns.length ? (
<HiddenColumnsBar type="button" onClick={handleShowAllColumns}>
@@ -386,16 +716,16 @@ function DataTableModalFactory(
</StyledConfigureButton>
{datasets[dataId] ? (
<DataTable
- key={dataId}
+ key={`${dataId}-${showFiltered ? 'filtered' : 'all'}`}
dataId={dataId}
columns={columns}
colMeta={colMeta}
cellSizeCache={cellSizeCache}
- dataContainer={activeDataset.dataContainer}
+ dataContainer={displayData.dataContainer}
pinnedColumns={(activeDataset.pinnedColumns || []).filter(
column => !hiddenColumns.includes(column)
)}
- sortOrder={activeDataset.sortOrder}
+ sortOrder={displayData.sortOrder}
sortColumn={activeDataset.sortColumn || DEFAULT_SORT_COLUMN}
copyTableColumn={handleCopyTableColumn}
pinTableColumn={handlePinTableColumn}
@@ -412,7 +742,7 @@ function DataTableModalFactory(
canCreateFilter && createFilterFromColumn ? handleCreateFilterFromColumn : undefined
}
hasStats={enableColumnStats}
- loadColumnStats={enableColumnStats ? loadColumnStats : undefined}
+ loadColumnStats={enableColumnStats ? handleLoadColumnStats : undefined}
/>
) : null}
</TableContainer>
diff --git a/src/localization/src/translations/en.ts b/src/localization/src/translations/en.ts
index cca7702b1..7ad9771d9 100644
--- a/src/localization/src/translations/en.ts
+++ b/src/localization/src/translations/en.ts
@@ -760,6 +760,12 @@ export default {
rowCount: '{rowCount} Rows',
tiledDatasetWarning: "* Export Data for Tiled datasets isn't supported"
},
+ dataTable: {
+ show: 'Show',
+ allData: 'All Data',
+ filteredData: 'Filtered Data',
+ rowCount: '{rowCount, plural, one {# row} other {# rows}}'
+ },
deleteData: {
warning: 'you are going to delete this dataset. It will affect {length} layers'
},
diff --git a/test/browser/components/modals/data-table-modal-test.js b/test/browser/components/modals/data-table-modal-test.js
index a2e73ad3d..30ca66d2a 100644
--- a/test/browser/components/modals/data-table-modal-test.js
+++ b/test/browser/components/modals/data-table-modal-test.js
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import React from 'react';
+import React, {useState} from 'react';
import {act} from 'react-dom/test-utils';
import test from 'tape-catch';
import global from 'global';
@@ -39,6 +39,10 @@ const DataTableModal = appInjector.get(DataTableModalFactory);
const DataTable = appInjector.get(DataTableFactory);
const FieldToken = appInjector.get(FieldTokenFactory);
+function mountDataTableModal(node) {
+ return mountWithTheme(<IntlWrapper>{node}</IntlWrapper>);
+}
+
const expectedCellSizeCache = {
'gps_data.utc_timestamp': {row: 145, header: 150},
'gps_data.lat': {row: 96, header: 108},
@@ -198,7 +202,7 @@ test('Components -> DataTableModal.render: csv 1', t => {
mountWithTheme(<DataTableModal />);
}, 'Show not fail without data');
- const wrapper = mountWithTheme(
+ const wrapper = mountDataTableModal(
<DataTableModal datasets={StateWFiles.visState.datasets} dataId={testCsvDataId} />
);
@@ -275,7 +279,7 @@ test('Components -> DataTableModal.render: csv 1', t => {
test('Components -> DataTableModal.render: disableColumnStats', t => {
initApplicationConfig({enableColumnStats: false});
try {
- const wrapper = mountWithTheme(
+ const wrapper = mountDataTableModal(
<DataTableModal datasets={StateWFiles.visState.datasets} dataId={testCsvDataId} />
);
const props = wrapper.find(DataTable).at(0).props();
@@ -289,7 +293,7 @@ test('Components -> DataTableModal.render: disableColumnStats', t => {
test('Components -> DataTableModal -> click tab', t => {
const showDatasetTable = sinon.spy();
- const wrapper = mountWithTheme(
+ const wrapper = mountDataTableModal(
<DataTableModal
datasets={StateWFiles.visState.datasets}
dataId={testCsvDataId}
@@ -325,8 +329,428 @@ test('Components -> DataTableModal -> click tab', t => {
t.end();
});
+test('Components -> DataTableModal -> toggle filtered data', t => {
+ const fields = [{name: 'name', type: 'string', displayName: 'name', fieldIdx: 0}];
+ const rows = [['a'], ['b'], ['c'], ['d']];
+ const dataContainer = createDataContainer(rows, {fields});
+ const otherContainer = createDataContainer([['x'], ['y']], {fields});
+ const sortOrder = [3, 1, 0, 2];
+
+ const smoothie = {
+ id: 'smoothie',
+ label: 'smoothie',
+ color: [113, 113, 113],
+ dataContainer,
+ fields,
+ allIndexes: [0, 1, 2, 3],
+ sortOrder
+ };
+ const other = {
+ id: 'other',
+ label: 'other',
+ color: [10, 10, 10],
+ dataContainer: otherContainer,
+ fields,
+ allIndexes: [0, 1],
+ sortOrder: null
+ };
+
+ const applyCalls = [];
+
+ const FilteredHarness = () => {
+ const [datasets, setDatasets] = useState({smoothie, other});
+ const [dataId, setDataId] = useState('smoothie');
+ const applyCPUFilter = id => {
+ applyCalls.push(id);
+ setDatasets(current => ({
+ ...current,
+ [id]: {
+ ...current[id],
+ filteredIdxCPU:
+ id === 'smoothie' ? (current[id].filterRecord?.version === 2 ? [2] : [0, 1]) : [0]
+ }
+ }));
+ };
+
+ return (
+ <div>
+ <button
+ className="change-filter"
+ onClick={() =>
+ setDatasets(current => ({
+ ...current,
+ smoothie: {...current.smoothie, filterRecord: {version: 2}}
+ }))
+ }
+ />
+ <DataTableModal
+ datasets={datasets}
+ dataId={dataId}
+ showDatasetTable={setDataId}
+ applyCPUFilter={applyCPUFilter}
+ />
+ </div>
+ );
+ };
+
+ const wrapper = mountDataTableModal(<FilteredHarness />);
+
+ t.deepEqual(applyCalls, [], 'should not filter until filtered data is selected');
+ t.equal(
+ wrapper.find('.data-view-label').first().text(),
+ 'Show',
+ 'should label the data view control'
+ );
+ t.equal(wrapper.find('.data-view-all').first().text(), 'All Data', 'should offer all data');
+ t.equal(
+ wrapper.find('.data-view-filtered').first().text(),
+ 'Filtered Data',
+ 'should offer filtered data'
+ );
+ t.equal(
+ wrapper.find('.data-view-all').first().prop('selected'),
+ true,
+ 'all data starts selected'
+ );
+ t.equal(wrapper.find('.data-view-row-count').first().text(), '4 rows', 'shows every row');
+ t.equal(
+ wrapper.find(DataTable).first().prop('dataContainer'),
+ dataContainer,
+ 'table reads the full dataset'
+ );
+ t.deepEqual(wrapper.find(DataTable).first().prop('sortOrder'), sortOrder, 'keeps the table sort');
+
+ wrapper.find('.data-view-filtered').first().simulate('click');
+
+ t.deepEqual(applyCalls, ['smoothie'], 'requests a CPU filter for the active dataset');
+ t.equal(
+ wrapper.find('.data-view-filtered').first().prop('selected'),
+ true,
+ 'filtered data is selected'
+ );
+ t.equal(
+ wrapper.find('.data-view-row-count').first().text(),
+ '2 rows',
+ 'shows the filtered row count'
+ );
+
+ const filteredContainer = wrapper.find(DataTable).first().prop('dataContainer');
+ t.equal(filteredContainer.numRows(), 2, 'table has one row per filtered index');
+ t.equal(filteredContainer.valueAt(0, 0), 'b', 'first filtered row follows the active sort');
+ t.equal(filteredContainer.valueAt(1, 0), 'a', 'second filtered row follows the active sort');
+ t.equal(
+ wrapper.find(DataTable).first().prop('sortOrder'),
+ null,
+ 'filtered rows are ordered by the indexed container'
+ );
+
+ wrapper.find('.change-filter').simulate('click');
+ t.deepEqual(applyCalls, ['smoothie', 'smoothie'], 'refreshes the CPU filter when filters change');
+ t.equal(
+ wrapper.find(DataTable).first().prop('dataContainer').valueAt(0, 0),
+ 'c',
+ 'shows rows from the updated filter'
+ );
+
+ wrapper.find('.data-view-all').first().simulate('click');
+ t.deepEqual(applyCalls, ['smoothie', 'smoothie'], 'all data does not request another filter');
+ t.equal(
+ wrapper.find('.data-view-row-count').first().text(),
+ '4 rows',
+ 'restores the full row count'
+ );
+ t.equal(
+ wrapper.find(DataTable).first().prop('dataContainer'),
+ dataContainer,
+ 'restores the full dataset'
+ );
+
+ wrapper.find('.data-view-filtered').first().simulate('click');
+ wrapper.find(DatasetModalTab).at(1).simulate('click');
+
+ t.deepEqual(
+ applyCalls,
+ ['smoothie', 'smoothie', 'smoothie', 'other'],
+ 'filters the dataset opened from the tabs'
+ );
+ t.equal(
+ wrapper.find('.data-view-row-count').first().text(),
+ '1 row',
+ 'counts the newly filtered dataset'
+ );
+ t.equal(
+ wrapper.find(DataTable).first().prop('dataContainer').valueAt(0, 0),
+ 'x',
+ 'shows the filtered row from the selected dataset'
+ );
+
+ t.end();
+});
+
+test('Components -> DataTableModal -> filter when the active dataset changes', t => {
+ const fields = [{name: 'name', type: 'string', displayName: 'name', fieldIdx: 0}];
+ const smoothieContainer = createDataContainer([['a'], ['b'], ['c']], {fields});
+ const otherContainer = createDataContainer([['x'], ['y']], {fields});
+ const applyCalls = [];
+
+ const ExternalSwitchHarness = () => {
+ const [dataId, setDataId] = useState('smoothie');
+ const [datasets, setDatasets] = useState({
+ smoothie: {
+ id: 'smoothie',
+ label: 'smoothie',
+ color: [113, 113, 113],
+ dataContainer: smoothieContainer,
+ fields
+ },
+ other: {
+ id: 'other',
+ label: 'other',
+ color: [10, 10, 10],
+ dataContainer: otherContainer,
+ fields
+ }
+ });
+
+ return (
+ <div>
+ <button className="switch-dataset" onClick={() => setDataId('other')} />
+ <DataTableModal
+ datasets={datasets}
+ dataId={dataId}
+ showDatasetTable={() => {}}
+ applyCPUFilter={id => {
+ applyCalls.push(id);
+ setDatasets(current => ({
+ ...current,
+ [id]: {
+ ...current[id],
+ filteredIdxCPU: [1]
+ }
+ }));
+ }}
+ />
+ </div>
+ );
+ };
+
+ const wrapper = mountDataTableModal(<ExternalSwitchHarness />);
+ wrapper.find('.data-view-filtered').first().simulate('click');
+ wrapper.find('.switch-dataset').simulate('click');
+
+ t.deepEqual(
+ applyCalls,
+ ['smoothie', 'other'],
+ 'applies the CPU filter when the active dataset changes'
+ );
+ t.equal(wrapper.find('.data-view-row-count').first().text(), '1 row', 'counts the new dataset');
+ t.equal(
+ wrapper.find(DataTable).first().prop('dataContainer').valueAt(0, 0),
+ 'y',
+ 'shows the filtered row from the newly active dataset'
+ );
+
+ t.end();
+});
+
+test('Components -> DataTableModal -> filtered column stats', async t => {
+ const allValueStats = {
+ type: 'numeric',
+ mean: 100,
+ std: 0,
+ percentNulls: 0,
+ bins: [],
+ quantiles: []
+ };
+ const allScoreStats = {
+ type: 'numeric',
+ mean: 999,
+ std: 0,
+ percentNulls: 0,
+ bins: [],
+ quantiles: []
+ };
+ const fields = [
+ {
+ name: 'value',
+ displayName: 'value',
+ type: 'integer',
+ fieldIdx: 0,
+ filterProps: {columnStats: allValueStats}
+ },
+ {
+ name: 'score',
+ displayName: 'score',
+ type: 'integer',
+ fieldIdx: 1,
+ filterProps: {columnStats: allScoreStats, mappedValue: [2, 4, 9]}
+ }
+ ];
+ const dataContainer = createDataContainer(
+ [
+ [1, 2],
+ [null, 4],
+ [100, 9]
+ ],
+ {fields}
+ );
+ const loadColumnStats = sinon.spy();
+ const filteredIndexes = [0, 1];
+
+ const StatsHarness = () => {
+ const [datasets, setDatasets] = useState({
+ smoothie: {
+ id: 'smoothie',
+ label: 'smoothie',
+ color: [1, 1, 1],
+ dataContainer,
+ fields
+ }
+ });
+
+ return (
+ <DataTableModal
+ datasets={datasets}
+ dataId="smoothie"
+ applyCPUFilter={() => {
+ setDatasets(current => ({
+ smoothie: {...current.smoothie, filteredIdxCPU: filteredIndexes}
+ }));
+ }}
+ loadColumnStats={loadColumnStats}
+ />
+ );
+ };
+
+ const wrapper = mountDataTableModal(<StatsHarness />);
+ const allMeta = wrapper.find(DataTable).first().prop('colMeta');
+ t.equal(allMeta.value.columnStats, allValueStats, 'all data keeps the stored column stats');
+ t.equal(allMeta.score.columnStats, allScoreStats, 'all data keeps mapped column stats');
+
+ wrapper.find('.data-view-filtered').first().simulate('click');
+ const pendingMeta = wrapper.find(DataTable).first().prop('colMeta');
+ t.equal(pendingMeta.value.columnStats, undefined, 'filtered view drops all-data column stats');
+ t.equal(pendingMeta.value.isLoadingStats, false, 'filtered stats load when the header asks');
+
+ await act(async () => {
+ wrapper.find(DataTable).first().prop('loadColumnStats')('smoothie', ['value', 'score']);
+ await new Promise(resolve => setTimeout(resolve, 0));
+ });
+ wrapper.update();
+
+ const filteredMeta = wrapper.find(DataTable).first().prop('colMeta');
+ t.equal(filteredMeta.value.columnStats.mean, 1, 'numeric stats use filtered rows');
+ t.equal(
+ filteredMeta.value.columnStats.percentNulls,
+ 0.5,
+ 'nulls are counted in the filtered rows'
+ );
+ t.equal(filteredMeta.score.columnStats.mean, 3, 'stats use mapped values for filtered rows');
+ t.equal(filteredMeta.value.isLoadingStats, false, 'filtered stats finish loading');
+ t.equal(loadColumnStats.called, false, 'filtered stats do not replace the stored all-data stats');
+ t.equal(fields[0].filterProps.columnStats, allValueStats, 'stored value stats stay on the field');
+ t.equal(fields[1].filterProps.columnStats, allScoreStats, 'stored score stats stay on the field');
+
+ wrapper.find('.data-view-all').first().simulate('click');
+ const restoredMeta = wrapper.find(DataTable).first().prop('colMeta');
+ t.equal(restoredMeta.value.columnStats, allValueStats, 'all data shows the stored stats again');
+ t.equal(
+ restoredMeta.score.columnStats,
+ allScoreStats,
+ 'all data shows the stored mapped stats again'
+ );
+
+ wrapper.find(DataTable).first().prop('loadColumnStats')('smoothie', 'value');
+ t.ok(
+ loadColumnStats.calledWith('smoothie', 'value'),
+ 'all data still loads stats through the dataset action'
+ );
+
+ t.end();
+});
+
+test('Components -> DataTableModal -> filtered data before indexes are ready', t => {
+ const fields = [{name: 'name', type: 'string', displayName: 'name', fieldIdx: 0}];
+ const dataContainer = createDataContainer([['a'], ['b']], {fields});
+ const applyCPUFilter = sinon.spy();
+
+ const wrapper = mountDataTableModal(
+ <DataTableModal
+ datasets={{
+ smoothie: {
+ id: 'smoothie',
+ label: 'smoothie',
+ color: [1, 1, 1],
+ dataContainer,
+ fields
+ }
+ }}
+ dataId="smoothie"
+ applyCPUFilter={applyCPUFilter}
+ />
+ );
+
+ wrapper.find('.data-view-filtered').first().simulate('click');
+
+ t.ok(applyCPUFilter.calledOnce, 'requests a CPU filter');
+ t.ok(applyCPUFilter.calledWith('smoothie'), 'filters the active dataset');
+ t.equal(
+ wrapper.find('.data-view-row-count').first().text(),
+ '-',
+ 'row count waits until filtered indexes exist'
+ );
+ t.equal(
+ wrapper.find(DataTable).first().prop('dataContainer').numRows(),
+ 0,
+ 'does not list unfiltered rows while the filter is pending'
+ );
+
+ t.end();
+});
+
+test('Components -> DataTableModal -> filtered data with no matching rows', t => {
+ const fields = [{name: 'name', type: 'string', displayName: 'name', fieldIdx: 0}];
+ const dataContainer = createDataContainer([['a'], ['b']], {fields});
+
+ const EmptyFilterHarness = () => {
+ const [datasets, setDatasets] = useState({
+ smoothie: {
+ id: 'smoothie',
+ label: 'smoothie',
+ color: [1, 1, 1],
+ dataContainer,
+ fields
+ }
+ });
+
+ return (
+ <DataTableModal
+ datasets={datasets}
+ dataId="smoothie"
+ applyCPUFilter={() => {
+ setDatasets(current => ({
+ smoothie: {...current.smoothie, filteredIdxCPU: []}
+ }));
+ }}
+ />
+ );
+ };
+
+ const wrapper = mountDataTableModal(<EmptyFilterHarness />);
+ wrapper.find('.data-view-filtered').first().simulate('click');
+
+ t.equal(wrapper.find('.data-view-row-count').first().text(), '0 rows', 'shows a zero row count');
+ t.equal(
+ wrapper.find(DataTable).first().prop('dataContainer').numRows(),
+ 0,
+ 'table is empty when every row is filtered out'
+ );
+
+ t.end();
+});
+
test('Components -> DataTableModal -> render DataTable: csv 1', t => {
- const wrapper = mountWithTheme(
+ const wrapper = mountDataTableModal(
<DataTableModal datasets={StateWFiles.visState.datasets} dataId={testCsvDataId} />
);
t.equal(wrapper.find(DataTable).length, 1, 'should render 1 DataTable');
@@ -461,7 +885,7 @@ test('Components -> DataTableModal -> render DataTable: sort, pin and display fo
VisStateActions.pinTableColumn(testCsvDataId, 'has_result')
);
- const wrapper = mountWithTheme(
+ const wrapper = mountDataTableModal(
<DataTableModal datasets={nextState2.datasets} dataId={testCsvDataId} />
);
@@ -539,7 +963,7 @@ test('Components -> DatableModal -> sort/pin/copy and display format should be c
const setColumnDisplayFormat = sinon.spy();
const column = 'gps_data.lat';
- const wrapper = mountWithTheme(
+ const wrapper = mountDataTableModal(
<DataTableModal
datasets={initialState.datasets}
dataId={testCsvDataId}
@@ -602,7 +1026,7 @@ test('Components -> cellSize -> renderedSize', t => {
];
const dataContainer = createDataContainer(texts, {fields});
- const wrapper = mountWithTheme(
+ const wrapper = mountDataTableModal(
<DataTableModal
datasets={{
smoothie: {
@@ -647,7 +1071,7 @@ test('Components -> DataTableModal.render: csv 2', t => {
const fieldsWithDisplayFormat = geoStyleFields;
fieldsWithDisplayFormat[5].displayFormat = TOOLTIP_FORMATS.DECIMAL_SCIENTIFIC_FIXED_2.format;
- const wrapper = mountWithTheme(
+ const wrapper = mountDataTableModal(
<DataTableModal
datasets={{
smoothie: {