Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

feat: add column statistics to the data table (#3646)

Igor Dykhta34fd0bfc4999
diff --git a/src/actions/src/action-types.ts b/src/actions/src/action-types.ts
index ab2d20daf..9d1005e53 100644
--- a/src/actions/src/action-types.ts
+++ b/src/actions/src/action-types.ts
@@ -109,6 +109,9 @@ export const ActionTypes = {
PIN_TABLE_COLUMN: `${ACTION_PREFIX}PIN_TABLE_COLUMN`,
COPY_TABLE_COLUMN: `${ACTION_PREFIX}COPY_TABLE_COLUMN`,
SET_COLUMN_DISPLAY_FORMAT: `${ACTION_PREFIX}SET_COLUMN_DISPLAY_FORMAT`,
+ LOAD_COLUMN_STATS: `${ACTION_PREFIX}LOAD_COLUMN_STATS`,
+ LOAD_COLUMN_STATS_SUCCESS: `${ACTION_PREFIX}LOAD_COLUMN_STATS_SUCCESS`,
+ LOAD_COLUMN_STATS_ERROR: `${ACTION_PREFIX}LOAD_COLUMN_STATS_ERROR`,
NEXT_FILE_BATCH: `${ACTION_PREFIX}NEXT_FILE_BATCH`,
PROCESS_FILE_CONTENT: `${ACTION_PREFIX}PROCESS_FILE_CONTENT`,
UPDATE_TABLE_COLOR: `${ACTION_PREFIX}UPDATE_TABLE_COLOR`,
diff --git a/src/actions/src/vis-state-actions.ts b/src/actions/src/vis-state-actions.ts
index 1e5a1b4ca..3dd07777b 100644
--- a/src/actions/src/vis-state-actions.ts
+++ b/src/actions/src/vis-state-actions.ts
@@ -1044,6 +1044,83 @@ export function setColumnDisplayFormat(
};
}
+export type LoadColumnStatsUpdaterAction = {
+ dataId: string;
+ fieldName: string | string[];
+};
+
+/**
+ * Lazily compute column statistics for the data table header.
+ * @param dataId
+ * @param fieldName one or more field names
+ * @returns action
+ * @public
+ */
+export function loadColumnStats(
+ dataId: string,
+ fieldName: string | string[]
+): Merge<LoadColumnStatsUpdaterAction, {type: typeof ActionTypes.LOAD_COLUMN_STATS}> {
+ return {
+ type: ActionTypes.LOAD_COLUMN_STATS,
+ dataId,
+ fieldName
+ };
+}
+
+export type LoadColumnStatsSuccessUpdaterAction = {
+ dataId: string;
+ fieldName: string;
+ result: any;
+ filterProps?: any;
+};
+
+/**
+ * Column statistics finished loading
+ * @returns action
+ * @public
+ */
+export function loadColumnStatsSuccess(
+ dataId: string,
+ fieldName: string,
+ result: any,
+ filterProps?: any
+): Merge<
+ LoadColumnStatsSuccessUpdaterAction,
+ {type: typeof ActionTypes.LOAD_COLUMN_STATS_SUCCESS}
+> {
+ return {
+ type: ActionTypes.LOAD_COLUMN_STATS_SUCCESS,
+ dataId,
+ fieldName,
+ result,
+ filterProps
+ };
+}
+
+export type LoadColumnStatsErrorUpdaterAction = {
+ dataId: string;
+ fieldName: string;
+ error: Error;
+};
+
+/**
+ * Column statistics failed to load
+ * @returns action
+ * @public
+ */
+export function loadColumnStatsError(
+ dataId: string,
+ fieldName: string,
+ error: Error
+): Merge<LoadColumnStatsErrorUpdaterAction, {type: typeof ActionTypes.LOAD_COLUMN_STATS_ERROR}> {
+ return {
+ type: ActionTypes.LOAD_COLUMN_STATS_ERROR,
+ dataId,
+ fieldName,
+ error
+ };
+}
+
export type AddDataToMapUpdaterOptions = {
centerMap?: boolean;
padding?: ViewportPadding;
diff --git a/src/components/src/common/data-table/cell-size.ts b/src/components/src/common/data-table/cell-size.ts
index 90b0faf0f..3b2686f64 100644
--- a/src/components/src/common/data-table/cell-size.ts
+++ b/src/components/src/common/data-table/cell-size.ts
@@ -13,6 +13,14 @@ const EDGE_COLUMN_PADDING = 10;
// be added to min cell width in order to show column name
const MAX_EMPTY_COLUMN_SPACE = 60;
+const GEOMETRY_FIELD_TYPES: Record<string, boolean> = {
+ geojson: true,
+ geoarrow: true
+};
+
+// GeoJSON payloads can be huge; do not size the column from cell text.
+const MAX_GEOMETRY_CELL_SIZE = 180;
+
type RenderSizeParam = {
text: {dataContainer: DataContainerInterface; column: string};
type?: string;
@@ -63,25 +71,32 @@ export function renderedSize({
const context = textCanvas.getContext('2d');
context.font = [fontSize, font].join('px ');
- let rowsToSample = [...Array(numRowsToCalculate)].map(() =>
- Math.floor(Math.random() * (dataContainer.numRows() - 1))
- );
+ const isGeometryColumn = Boolean(GEOMETRY_FIELD_TYPES[type]);
+ const maxWidth = isGeometryColumn ? MAX_GEOMETRY_CELL_SIZE : maxCellSize;
+
+ let rowWidth = minCellSize + cellPadding;
+ if (!isGeometryColumn) {
+ let rowsToSample = [...Array(numRowsToCalculate)].map(() =>
+ Math.floor(Math.random() * Math.max(dataContainer.numRows() - 1, 0))
+ );
- // If we have less than 10 rows, lets measure all of them
- if (dataContainer.numRows() <= numRowsToCalculate) {
- rowsToSample = Array.from(Array(dataContainer.numRows()).keys());
+ // If we have less than 10 rows, lets measure all of them
+ if (dataContainer.numRows() <= numRowsToCalculate) {
+ rowsToSample = Array.from(Array(dataContainer.numRows()).keys());
+ }
+ rowWidth = Math.max(
+ rowWidth,
+ ...rowsToSample.map(rowIdx => {
+ const value = parseFieldValue(dataContainer.valueAt(rowIdx, colIdx), type);
+ // measuring large text cause slow performance
+ if (value.length > maxWidth) {
+ return maxWidth;
+ }
+ const textWidth = context.measureText(value).width;
+ return Math.ceil(textWidth) + cellPadding;
+ })
+ );
}
- const rowWidth = Math.max(
- ...rowsToSample.map(rowIdx => {
- const value = parseFieldValue(dataContainer.valueAt(rowIdx, colIdx), type);
- // measuring large text cause slow performance
- if (value.length > maxCellSize) {
- return maxCellSize;
- }
- const textWidth = context.measureText(value).width;
- return Math.ceil(textWidth) + cellPadding;
- })
- );
// header cell only has left padding
const headerWidth =
Math.ceil(context.measureText(column).width) + cellPadding / 2 + optionsButton;
@@ -91,7 +106,7 @@ export function renderedSize({
// min header width is measured by cell
const minHeaderWidth = minCellSize + cellPadding / 2 + optionsButton;
- const clampedRowWidth = clamp(minRowWidth, maxCellSize, rowWidth);
+ const clampedRowWidth = clamp(minRowWidth, maxWidth, rowWidth);
const clampedHeaderWidth = clamp(minHeaderWidth, maxHeaderSize, headerWidth);
// cleanup
@@ -154,14 +169,17 @@ function expandCellSize(
let remaining = roomToFill;
const expandedCellSize = columnOrder.reduce((accu, col) => {
- let size = cellSizeCache[col].row;
- if (cellSizeCache[col].row < cellSizeCache[col].header && remaining > 0) {
+ const colSize = cellSizeCache[col];
+ if (!colSize) {
+ return accu;
+ }
+
+ let size = colSize.row;
+ if (colSize.row < colSize.header && remaining > 0) {
// if we are cutting off the header, expand to fit it
size =
- cellSizeCache[col].header - cellSizeCache[col].row < remaining
- ? cellSizeCache[col].header
- : cellSizeCache[col].row + remaining;
- remaining -= size - cellSizeCache[col].row;
+ colSize.header - colSize.row < remaining ? colSize.header : colSize.row + remaining;
+ remaining -= size - colSize.row;
}
return {
@@ -172,9 +190,10 @@ function expandCellSize(
let ghost: number | null = null;
if (remaining > 0 && remaining < MIN_GHOST_CELL_SIZE) {
- // expand last cell
const lastCell = columnOrder[columnOrder.length - 1];
- expandedCellSize[lastCell] += remaining;
+ if (lastCell && expandedCellSize[lastCell] != null) {
+ expandedCellSize[lastCell] += remaining;
+ }
} else if (remaining >= MIN_GHOST_CELL_SIZE) {
// if too much left add a ghost cell
ghost = remaining;
diff --git a/src/components/src/common/data-table/column-statistics-categoric.tsx b/src/components/src/common/data-table/column-statistics-categoric.tsx
new file mode 100644
index 000000000..a5b711912
--- /dev/null
+++ b/src/components/src/common/data-table/column-statistics-categoric.tsx
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import React, {useMemo} from 'react';
+import styled from 'styled-components';
+
+import {CategoricalColumnStats, isNumber} from '@kepler.gl/utils';
+
+import {
+ ColumnStatisticsContainer,
+ StatisticRowContainer,
+ StatisticRow,
+ StyledStatsName,
+ TopPlotContainer,
+ STATS_MARGIN,
+ STATS_WIDTH,
+ Margin,
+ formatPercent
+} from './column-statistics-components';
+
+const StyledHighlightNumber = styled.div`
+ font-size: 24px;
+ color: ${props => props.theme.activeColorLT};
+ width: 100%;
+ text-align: center;
+`;
+
+export type ColumnStatisticsCategoricalProps = CategoricalColumnStats & {
+ width?: number;
+ margin?: Margin;
+};
+
+const ColumnStatisticsCategorical: React.FC<ColumnStatisticsCategoricalProps> = ({
+ width = STATS_WIDTH,
+ margin = STATS_MARGIN,
+ uniqueValues,
+ percentNulls
+}) => {
+ const nullsRight = useMemo(
+ () => (isNumber(percentNulls) ? formatPercent(percentNulls) : null),
+ [percentNulls]
+ );
+ return (
+ <ColumnStatisticsContainer width={width} margin={margin}>
+ <TopPlotContainer>
+ <StatisticRowContainer>
+ <StyledStatsName>Unique Values</StyledStatsName>
+ </StatisticRowContainer>
+ <StyledHighlightNumber>{uniqueValues}</StyledHighlightNumber>
+ </TopPlotContainer>
+ <StatisticRow
+ left="Nulls"
+ isBad={isNumber(percentNulls) && percentNulls > 0}
+ right={nullsRight}
+ />
+ </ColumnStatisticsContainer>
+ );
+};
+
+export default ColumnStatisticsCategorical;
diff --git a/src/components/src/common/data-table/column-statistics-components.tsx b/src/components/src/common/data-table/column-statistics-components.tsx
new file mode 100644
index 000000000..a09a34b1e
--- /dev/null
+++ b/src/components/src/common/data-table/column-statistics-components.tsx
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import React from 'react';
+import styled from 'styled-components';
+
+export type Margin = {
+ top: number;
+ left: number;
+ right: number;
+ bottom: number;
+};
+
+export const STATS_WIDTH = 180;
+export const STATS_MARGIN: Margin = {top: 10, left: 10, right: 10, bottom: 10};
+export const HISTOGRAM_MARGIN: Margin = {top: 10, left: 0, right: 0, bottom: 10};
+export const HISTOGRAM_HEIGHT = 50;
+
+type ColumnStatisticsContainerProps = {
+ width: number;
+ margin: Margin;
+};
+
+export const ColumnStatisticsContainer = styled.div.withConfig({
+ shouldForwardProp: prop => prop !== 'width' && prop !== 'margin'
+})<ColumnStatisticsContainerProps>`
+ width: 100%;
+ box-sizing: border-box;
+ padding-top: ${props => props.margin.top}px;
+ padding-bottom: ${props => props.margin.bottom}px;
+ padding-left: ${props => props.margin.left}px;
+ padding-right: ${props => props.margin.right}px;
+ hr {
+ border: 0;
+ border-top: 1px solid ${props => props.theme.cellBorderColor};
+ margin: 8px 0;
+ }
+`;
+
+export const StatisticRowContainer = styled.div`
+ display: flex;
+ justify-content: space-between;
+ align-items: baseline;
+ column-gap: 8px;
+ flex-wrap: wrap;
+ margin-bottom: 8px;
+ font-size: ${props => props.theme.cellFontSize}px;
+ line-height: 1.2;
+ &:last-child {
+ margin-bottom: 0;
+ }
+`;
+
+type StyledStatsNameProps = {
+ $isBad?: boolean;
+};
+
+export const StyledStatsName = styled.div<StyledStatsNameProps>`
+ flex-shrink: 0;
+ color: ${props =>
+ props.$isBad ? props.theme.notificationColors?.error || '#f25138' : props.theme.subtextColor};
+`;
+
+export const StyledStatsValue = styled.div`
+ color: ${props => props.theme.dataTableTextColor};
+ white-space: nowrap;
+ text-align: right;
+ margin-left: auto;
+`;
+
+export const TopPlotContainer = styled.div`
+ margin-bottom: 12px;
+ height: ${HISTOGRAM_HEIGHT + HISTOGRAM_MARGIN.top + HISTOGRAM_MARGIN.bottom}px;
+`;
+
+type StatisticRowProps = {
+ left: string;
+ right: string | null;
+ isBad?: boolean;
+};
+
+export const StatisticRow: React.FC<StatisticRowProps> = ({left, isBad, right}) => (
+ <StatisticRowContainer>
+ <StyledStatsName $isBad={isBad}>{left}</StyledStatsName>
+ {typeof right === 'string' ? <StyledStatsValue>{right}</StyledStatsValue> : null}
+ </StatisticRowContainer>
+);
+
+export function formatPercent(n: number): string {
+ return `${(n * 100).toFixed(1)}%`;
+}
diff --git a/src/components/src/common/data-table/column-statistics-numeric.tsx b/src/components/src/common/data-table/column-statistics-numeric.tsx
new file mode 100644
index 000000000..6ab446739
--- /dev/null
+++ b/src/components/src/common/data-table/column-statistics-numeric.tsx
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import React, {useMemo} from 'react';
+
+import {NumericColumnStats, formatNumber, isNumber, roundToFour} from '@kepler.gl/utils';
+
+import HistogramPlotFactory, {HistogramPlotProps} from '../histogram-plot';
+import {
+ STATS_WIDTH,
+ STATS_MARGIN,
+ HISTOGRAM_MARGIN,
+ HISTOGRAM_HEIGHT,
+ TopPlotContainer,
+ ColumnStatisticsContainer,
+ StatisticRow,
+ formatPercent
+} from './column-statistics-components';
+
+export type ColumnStatisticsNumericProps = NumericColumnStats & {
+ width?: number;
+ margin?: {top: number; left: number; right: number; bottom: number};
+ type?: string;
+};
+
+const VALUE = [-Infinity, Infinity];
+
+ColumnStatisticsNumericFactory.deps = [HistogramPlotFactory];
+function ColumnStatisticsNumericFactory(
+ HistogramPlot: React.FC<HistogramPlotProps>
+): React.FC<ColumnStatisticsNumericProps> {
+ const ColumnStatisticsNumeric: React.FC<ColumnStatisticsNumericProps> = ({
+ bins,
+ quantiles = [],
+ mean,
+ std,
+ percentNulls,
+ width = STATS_WIDTH,
+ margin = STATS_MARGIN
+ }) => {
+ const histogramWidth = width - margin.left - margin.right;
+ const histogramsByGroup = useMemo(
+ () => ({
+ bins
+ }),
+ [bins]
+ );
+ const nullsRight = useMemo(
+ () => (isNumber(percentNulls) ? formatPercent(percentNulls) : null),
+ [percentNulls]
+ );
+ const meanRight = useMemo(() => (isNumber(mean) ? formatNumber(mean, 'real') : null), [mean]);
+ const stdRight = useMemo(() => (isNumber(std) ? formatNumber(std, 'real') : null), [std]);
+ const quantileDisplays = useMemo(
+ () =>
+ quantiles.map(row => ({
+ label: row.label,
+ value: isNumber(row.value)
+ ? String(Number.isInteger(row.value) ? row.value : roundToFour(row.value))
+ : null
+ })),
+ [quantiles]
+ );
+
+ return (
+ <ColumnStatisticsContainer width={width} margin={margin}>
+ <TopPlotContainer>
+ <HistogramPlot
+ histogramsByGroup={histogramsByGroup}
+ margin={HISTOGRAM_MARGIN}
+ width={histogramWidth}
+ height={HISTOGRAM_HEIGHT}
+ value={VALUE}
+ />
+ </TopPlotContainer>
+ <StatisticRow
+ left="Nulls"
+ isBad={isNumber(percentNulls) && percentNulls > 0}
+ right={nullsRight}
+ />
+ <StatisticRow left="Mean" right={meanRight} />
+ <StatisticRow left="Std. Dev." right={stdRight} />
+ <hr />
+ <StatisticRow left="Quantiles" right={null} />
+ {quantileDisplays.map((row, index) => (
+ <StatisticRow left={row.label} right={row.value} key={index} />
+ ))}
+ </ColumnStatisticsContainer>
+ );
+ };
+
+ return ColumnStatisticsNumeric;
+}
+export default ColumnStatisticsNumericFactory;
diff --git a/src/components/src/common/data-table/column-statistics-time.tsx b/src/components/src/common/data-table/column-statistics-time.tsx
new file mode 100644
index 000000000..2b72058f6
--- /dev/null
+++ b/src/components/src/common/data-table/column-statistics-time.tsx
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import React, {useMemo} from 'react';
+
+import {TimeColumnStats, datetimeFormatter, isNumber} from '@kepler.gl/utils';
+
+import HistogramPlotFactory, {HistogramPlotProps} from '../histogram-plot';
+import {
+ STATS_WIDTH,
+ STATS_MARGIN,
+ HISTOGRAM_MARGIN,
+ HISTOGRAM_HEIGHT,
+ StatisticRow,
+ ColumnStatisticsContainer,
+ TopPlotContainer,
+ formatPercent
+} from './column-statistics-components';
+
+export type ColumnStatisticsTimeProps = TimeColumnStats & {
+ width?: number;
+ margin?: {top: number; left: number; right: number; bottom: number};
+ type?: string;
+ format?: string;
+};
+
+const HISTOGRAM_VALUE = [-Infinity, Infinity];
+
+ColumnStatisticsTimeFactory.deps = [HistogramPlotFactory];
+function ColumnStatisticsTimeFactory(
+ HistogramPlot: React.FC<HistogramPlotProps>
+): React.FC<ColumnStatisticsTimeProps> {
+ const ColumnStatisticsTime: React.FC<ColumnStatisticsTimeProps> = ({
+ bins,
+ min,
+ max,
+ percentNulls,
+ width = STATS_WIDTH,
+ margin = STATS_MARGIN,
+ format
+ }) => {
+ const histogramWidth = width - margin.left - margin.right;
+ const histogramsByGroup = useMemo(
+ () => ({
+ bins
+ }),
+ [bins]
+ );
+
+ const nullsRight = useMemo(
+ () => (isNumber(percentNulls) ? formatPercent(percentNulls) : null),
+ [percentNulls]
+ );
+ const startRight = useMemo(
+ () => (isNumber(min) ? datetimeFormatter(null)(format)(min) : null),
+ [min, format]
+ );
+ const endRight = useMemo(
+ () => (isNumber(max) ? datetimeFormatter(null)(format)(max) : null),
+ [max, format]
+ );
+ return (
+ <ColumnStatisticsContainer width={width} margin={margin}>
+ <TopPlotContainer>
+ <HistogramPlot
+ histogramsByGroup={histogramsByGroup}
+ margin={HISTOGRAM_MARGIN}
+ width={histogramWidth}
+ height={HISTOGRAM_HEIGHT}
+ value={HISTOGRAM_VALUE}
+ />
+ </TopPlotContainer>
+ <StatisticRow
+ left="Nulls"
+ isBad={isNumber(percentNulls) && percentNulls > 0}
+ right={nullsRight}
+ />
+ <StatisticRow left="Start" right={startRight} />
+ <StatisticRow left="End" right={endRight} />
+ </ColumnStatisticsContainer>
+ );
+ };
+
+ return ColumnStatisticsTime;
+}
+export default ColumnStatisticsTimeFactory;
diff --git a/src/components/src/common/data-table/header-cell.tsx b/src/components/src/common/data-table/header-cell.tsx
index fca2c8688..82c1e49f7 100644
--- a/src/components/src/common/data-table/header-cell.tsx
+++ b/src/components/src/common/data-table/header-cell.tsx
@@ -1,38 +1,64 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import React, {CSSProperties, useState, useCallback} from 'react';
+import React, {CSSProperties, useState, useCallback, useEffect} from 'react';
import styled from 'styled-components';
import classnames from 'classnames';
import Button from './button';
import {ArrowUp, ArrowDown, VertThreeDots, Hash} from '../../common/icons';
import {SORT_ORDER} from '@kepler.gl/constants';
import OptionDropdown, {FormatterDropdown} from './option-dropdown';
-import {getFieldFormatLabels} from '@kepler.gl/utils';
+import {ColumnStatsType, getFieldFormatLabels} from '@kepler.gl/utils';
import {ColMeta} from '@kepler.gl/types';
import FieldTokenFactory, {FieldTokenProps} from '../../common/field-token';
import {DataTableProps} from './index';
+import LoadingSpinner from '../loading-spinner';
+import ColumnStatisticsCategorical from './column-statistics-categoric';
+import ColumnStatisticsNumericFactory, {
+ ColumnStatisticsNumericProps
+} from './column-statistics-numeric';
+import ColumnStatisticsTimeFactory, {ColumnStatisticsTimeProps} from './column-statistics-time';
-const StyledHeaderCell = styled.div`
+const StyledHeaderCell = styled.div<{$hasStats?: boolean; $firstCell?: boolean}>`
border-bottom: 1px solid ${props => props.theme.headerCellBorderColor};
border-top: 1px solid ${props => props.theme.headerCellBorderColor};
- padding-top: ${props => props.theme.headerPaddingTop}px;
+ border-right: ${props => (props.$hasStats ? `1px solid ${props.theme.cellBorderColor}` : 'none')};
+ padding-top: ${props => (props.$hasStats ? 0 : props.theme.headerPaddingTop)}px;
padding-right: 0;
- padding-bottom: ${props => props.theme.headerPaddingBottom}px;
- padding-left: ${props => props.theme.cellPaddingSide}px;
- align-items: center;
- justify-content: space-between;
+ padding-bottom: ${props => (props.$hasStats ? 0 : props.theme.headerPaddingBottom)}px;
+ padding-left: ${props => (props.$hasStats ? 0 : props.theme.cellPaddingSide)}px;
+ align-items: ${props => (props.$hasStats ? 'stretch' : 'center')};
+ justify-content: ${props => (props.$hasStats ? 'flex-start' : 'space-between')};
display: flex;
- flex-direction: row;
+ flex-direction: ${props => (props.$hasStats ? 'column' : 'row')};
background-color: ${props => props.theme.headerCellBackground};
+ overflow: hidden;
.n-sort-idx {
font-size: 9px;
}
+ .header-cell__top {
+ display: flex;
+ flex-direction: row;
+ justify-content: space-between;
+ align-items: center;
+ width: 100%;
+ height: ${props => props.theme.headerRowHeight}px;
+ flex: 0 0 auto;
+ box-sizing: border-box;
+ padding-top: ${props => props.theme.headerPaddingTop}px;
+ padding-bottom: ${props => props.theme.headerPaddingBottom}px;
+ padding-left: ${props =>
+ props.$firstCell
+ ? props.theme.cellPaddingSide + props.theme.edgeCellPaddingSide
+ : props.theme.cellPaddingSide}px;
+ }
.details {
font-weight: 500;
+ font-size: ${props => props.theme.cellFontSize}px;
display: flex;
- flex-direction: column;
+ flex-direction: row;
+ align-items: center;
justify-content: flex-start;
height: 100%;
overflow: hidden;
@@ -42,6 +68,9 @@ const StyledHeaderCell = styled.div`
display: flex;
align-items: center;
justify-content: space-between;
+ margin-left: 6px;
+ min-width: 0;
+ flex: 1;
.col-name__left {
display: flex;
@@ -71,6 +100,34 @@ const StyledHeaderCell = styled.div`
}
`;
+type StyledBottomStatsProps = {
+ $showStats?: boolean;
+};
+
+const StyledBottomStats = styled.div<StyledBottomStatsProps>`
+ background-color: ${props => props.theme.headerCellStatsBackground};
+ flex-grow: 1;
+ min-height: 0;
+ overflow: hidden;
+ opacity: ${props => (props.$showStats ? 1 : 0)};
+ transition: opacity 0.5s ease-in-out;
+ pointer-events: ${props => (props.$showStats ? 'auto' : 'none')};
+ width: 100%;
+`;
+
+const StyledLoading = styled.div`
+ width: 100%;
+ padding-top: 16px;
+ display: flex;
+ justify-content: center;
+`;
+
+const StyledStatsSpacer = styled.div`
+ height: ${props => props.theme.headerStatsControlHeight}px;
+ width: 100%;
+ flex-grow: 0;
+`;
+
type CellInfo = {
columnIndex: number;
isScrolling: boolean;
@@ -81,7 +138,7 @@ type CellInfo = {
style: CSSProperties;
};
-type HeaderCellProps = {
+export type HeaderCellProps = {
// passed down from react virtualized Grid
cellInfo: CellInfo;
columns: DataTableProps['columns'];
@@ -94,11 +151,33 @@ type HeaderCellProps = {
style: CSSProperties;
};
-const HeaderCellFactory = (FieldToken: React.FC<FieldTokenProps>) => {
+function HeaderCellFactory(
+ FieldToken: React.FC<FieldTokenProps>,
+ ColumnStatisticsNumeric: React.FC<ColumnStatisticsNumericProps>,
+ ColumnStatisticsTime: React.FC<ColumnStatisticsTimeProps>
+) {
+ const ColumnStatisticsSection: React.FC<{
+ columnStats: any;
+ type: string;
+ format?: string;
+ }> = ({columnStats, type, format}) => {
+ switch (columnStats.type) {
+ case ColumnStatsType.numeric:
+ return <ColumnStatisticsNumeric {...columnStats} type={type} />;
+ case ColumnStatsType.categorical:
+ return <ColumnStatisticsCategorical {...columnStats} />;
+ case ColumnStatsType.time:
+ return <ColumnStatisticsTime {...columnStats} format={format} />;
+ default:
+ return <div />;
+ }
+ };
+
const HeaderCell = ({
cellInfo,
columns,
isPinned,
+ showStats,
props,
toggleMoreOptions,
moreOptionsColumn
@@ -110,34 +189,125 @@ const HeaderCellFactory = (FieldToken: React.FC<FieldTokenProps>) => {
sortTableColumn,
pinTableColumn,
copyTableColumn,
- setColumnDisplayFormat
+ setColumnDisplayFormat,
+ hasStats,
+ dataId,
+ loadColumnStats
} = props;
const [showFormatter, setShowFormatter] = useState(false);
const column = columns[columnIndex];
- const isGhost = column.ghost;
- const isSorted = sortColumn[column];
+ const isGhost = Boolean(column.ghost);
+ const isSorted = isGhost ? false : sortColumn[column];
const firstCell = columnIndex === 0;
- const isFormatted = Boolean(colMeta[column]?.displayFormat);
- const formatLabels = isFormatted ? getFieldFormatLabels(colMeta[column].type) : [];
- const onSortTable = useCallback(() => sortTableColumn?.(column), [sortTableColumn, column]);
- const onToggleOptionMenu = useCallback(
- () => toggleMoreOptions(column),
- [toggleMoreOptions, column]
- );
- const onPin = useCallback(() => pinTableColumn(column), [pinTableColumn, column]);
- const onCopy = useCallback(() => copyTableColumn(column), [copyTableColumn, column]);
+ const {columnStats, isLoadingStats, type, format} = colMeta[isGhost ? '' : column] || {};
+ const isFormatted = isGhost ? false : Boolean(colMeta[column]?.displayFormat);
+ const formatLabels = isFormatted && !isGhost ? getFieldFormatLabels(colMeta[column].type) : [];
+
+ useEffect(() => {
+ if (
+ !isGhost &&
+ !columnStats &&
+ !isLoadingStats &&
+ dataId &&
+ showStats &&
+ hasStats &&
+ loadColumnStats
+ ) {
+ loadColumnStats(dataId, column);
+ }
+ }, [
+ isGhost,
+ columnStats,
+ isLoadingStats,
+ column,
+ dataId,
+ showStats,
+ hasStats,
+ loadColumnStats
+ ]);
+
+ const onSortTable = useCallback(() => {
+ if (!isGhost) sortTableColumn?.(column);
+ }, [sortTableColumn, isGhost, column]);
+ const onToggleOptionMenu = useCallback(() => {
+ if (!isGhost) toggleMoreOptions(column);
+ }, [toggleMoreOptions, isGhost, column]);
+ const onPin = useCallback(() => {
+ if (!isGhost) pinTableColumn(column);
+ }, [pinTableColumn, isGhost, column]);
+ const onCopy = useCallback(() => {
+ if (!isGhost) copyTableColumn(column);
+ }, [copyTableColumn, isGhost, column]);
const onSetDisplayFormat = useCallback(
displayFormat => {
- setColumnDisplayFormat?.({[column]: displayFormat.format});
+ if (!isGhost) setColumnDisplayFormat?.({[column]: displayFormat.format});
},
- [column, setColumnDisplayFormat]
+ [column, isGhost, setColumnDisplayFormat]
);
const onToggleDisplayFormat = useCallback(() => {
setShowFormatter(!showFormatter);
}, [showFormatter]);
+ const headerDetails = isGhost ? (
+ <div />
+ ) : (
+ <>
+ <section
+ className="details"
+ onClick={e => {
+ if (e.shiftKey && !isGhost) sortTableColumn?.(column);
+ }}
+ onDoubleClick={onSortTable}
+ >
+ <FieldToken type={colMeta[column].type} />
+ <div className="col-name">
+ <div className="col-name__left">
+ <div className="col-name__name">{colMeta[column].name}</div>
+ <Button className="col-name__sort" onClick={onSortTable}>
+ {isSorted ? (
+ isSorted === SORT_ORDER.ASCENDING ? (
+ <ArrowUp height="14px" />
+ ) : (
+ <ArrowDown height="14px" />
+ )
+ ) : null}
+ </Button>
+ <Button className="col-name__format" onClick={onToggleDisplayFormat}>
+ {isFormatted ? <Hash height="14px" /> : null}
+ <FormatterDropdown
+ left={0}
+ top={0}
+ isOpened={isFormatted && showFormatter}
+ displayFormat={colMeta[column].displayFormat}
+ setDisplayFormat={onSetDisplayFormat}
+ onClose={() => setShowFormatter(false)}
+ formatLabels={formatLabels}
+ />
+ </Button>
+ </div>
+ <Button className="more" onClick={onToggleOptionMenu}>
+ <VertThreeDots height="14px" />
+ </Button>
+ </div>
+ </section>
+
+ <section className="options">
+ <OptionDropdown
+ isOpened={moreOptionsColumn === column}
+ column={column}
+ colMeta={colMeta}
+ toggleMoreOptions={toggleMoreOptions}
+ sortTableColumn={sortTableColumn ? mode => sortTableColumn(column, mode) : undefined}
+ pinTableColumn={onPin}
+ copyTableColumn={onCopy}
+ setDisplayFormat={setColumnDisplayFormat ? onSetDisplayFormat : undefined}
+ />
+ </section>
+ </>
+ );
+
return (
<StyledHeaderCell
className={classnames('header-cell', {
@@ -147,69 +317,35 @@ const HeaderCellFactory = (FieldToken: React.FC<FieldTokenProps>) => {
})}
key={key}
style={style}
- onClick={e => {
- e.shiftKey ? sortTableColumn(column) : null;
- }}
- onDoubleClick={onSortTable}
- title={column}
+ $hasStats={Boolean(hasStats)}
+ $firstCell={firstCell}
+ title={isGhost ? undefined : column}
>
- {isGhost ? (
- <div />
+ {hasStats && !isGhost ? (
+ <div className="header-cell__top">{headerDetails}</div>
) : (
- <>
- <section className="details">
- <div className="col-name">
- <div className="col-name__left">
- <div className="col-name__name">{colMeta[column].name}</div>
- <Button className="col-name__sort" onClick={onSortTable}>
- {isSorted ? (
- isSorted === SORT_ORDER.ASCENDING ? (
- <ArrowUp height="14px" />
- ) : (
- <ArrowDown height="14px" />
- )
- ) : null}
- </Button>
- <Button className="col-name__format" onClick={onToggleDisplayFormat}>
- {isFormatted ? <Hash height="14px" /> : null}
- <FormatterDropdown
- left={0}
- top={0}
- isOpened={isFormatted && showFormatter}
- displayFormat={colMeta[column].displayFormat}
- setDisplayFormat={onSetDisplayFormat}
- onClose={() => setShowFormatter(false)}
- formatLabels={formatLabels}
- />
- </Button>
- </div>
- <Button className="more" onClick={onToggleOptionMenu}>
- <VertThreeDots height="14px" />
- </Button>
- </div>
- <FieldToken type={colMeta[column].type} />
- </section>
-
- <section className="options">
- <OptionDropdown
- isOpened={moreOptionsColumn === column}
- column={column}
- colMeta={colMeta}
- toggleMoreOptions={toggleMoreOptions}
- sortTableColumn={
- sortTableColumn ? mode => sortTableColumn(column, mode) : undefined
- }
- pinTableColumn={onPin}
- copyTableColumn={onCopy}
- setDisplayFormat={setColumnDisplayFormat ? onSetDisplayFormat : undefined}
- />
- </section>
- </>
+ headerDetails
)}
+ {hasStats && !isGhost ? <StyledStatsSpacer /> : null}
+ {hasStats && !isGhost ? (
+ <StyledBottomStats className="bottom-stats" $showStats={showStats}>
+ {isLoadingStats ? (
+ <StyledLoading>
+ <LoadingSpinner />
+ </StyledLoading>
+ ) : columnStats ? (
+ <ColumnStatisticsSection columnStats={columnStats} type={type} format={format} />
+ ) : null}
+ </StyledBottomStats>
+ ) : null}
</StyledHeaderCell>
);
};
return HeaderCell;
-};
-HeaderCellFactory.deps = [FieldTokenFactory];
+}
+HeaderCellFactory.deps = [
+ FieldTokenFactory,
+ ColumnStatisticsNumericFactory,
+ ColumnStatisticsTimeFactory
+];
export default HeaderCellFactory;
diff --git a/src/components/src/common/data-table/index.tsx b/src/components/src/common/data-table/index.tsx
index 46b22b4f8..23a1a39fb 100644
--- a/src/components/src/common/data-table/index.tsx
+++ b/src/components/src/common/data-table/index.tsx
@@ -9,6 +9,7 @@ import {createSelector} from 'reselect';
import get from 'lodash/get';
import debounce from 'lodash/debounce';
import {ArrowDown} from '../icons';
+import LoadingSpinner from '../loading-spinner';
import {CellSizeCache} from './cell-size';
@@ -24,6 +25,7 @@ import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
const defaultHeaderRowHeight = 55;
const defaultHeaderStatsControlHeight = 40;
const defaultRowHeight = 32;
+const STATS_TOGGLE_DURATION = '0.5s';
const overscanColumnCount = 10;
const overscanRowCount = 10;
// The default scrollbar width can range anywhere from 12px to 17px
@@ -45,11 +47,12 @@ const unpinnedClassList = {
type ContainerProps = {
$hasCustomScrollBarStyle?: boolean;
+ $hasStats?: boolean;
};
export const Container = styled.div<ContainerProps>`
display: flex;
- font-size: 11px;
+ font-size: ${props => props.theme.cellFontSize}px;
flex-grow: 1;
color: ${props => props.theme.dataTableTextColor};
width: 100%;
@@ -134,8 +137,6 @@ export const Container = styled.div<ContainerProps>`
height: 100%;
display: flex;
flex-direction: column;
- justify-content: center;
- align-items: flex-start;
text-align: center;
overflow: hidden;
// header border is rendered by header container
@@ -145,6 +146,8 @@ export const Container = styled.div<ContainerProps>`
}
}
.cell {
+ justify-content: center;
+ align-items: flex-start;
border-bottom: 1px solid ${props => props.theme.cellBorderColor};
border-right: 1px solid ${props => props.theme.cellBorderColor};
white-space: nowrap;
@@ -156,15 +159,27 @@ export const Container = styled.div<ContainerProps>`
text-decoration: none;
}
}
- .cell.end-cell,
- .header-cell.end-cell {
+ .header-cell {
+ justify-content: ${props => (props.$hasStats ? 'flex-start' : 'center')};
+ align-items: ${props => (props.$hasStats ? 'stretch' : 'flex-start')};
+ }
+ .cell.end-cell {
border-right: none;
padding-right: ${props => props.theme.cellPaddingSide + props.theme.edgeCellPaddingSide}px;
}
- .cell.first-cell,
- .header-cell.first-cell {
+ .header-cell.end-cell {
+ border-right: ${props =>
+ props.$hasStats ? `1px solid ${props.theme.cellBorderColor}` : 'none'};
+ padding-right: ${props =>
+ props.$hasStats ? 0 : props.theme.cellPaddingSide + props.theme.edgeCellPaddingSide}px;
+ }
+ .cell.first-cell {
padding-left: ${props => props.theme.cellPaddingSide + props.theme.edgeCellPaddingSide}px;
}
+ .header-cell.first-cell {
+ padding-left: ${props =>
+ props.$hasStats ? 0 : props.theme.cellPaddingSide + props.theme.edgeCellPaddingSide}px;
+ }
.cell.bottom-cell {
border-bottom: none;
}
@@ -178,6 +193,17 @@ export const Container = styled.div<ContainerProps>`
}
`;
+const StyledTableLoading = styled.div.attrs({
+ className: 'data-table-loading'
+})`
+ display: flex;
+ flex-grow: 1;
+ align-items: center;
+ justify-content: center;
+ width: 100%;
+ min-height: 240px;
+`;
+
const defaultColumnWidth = 200;
export type SortColumn = {
@@ -193,6 +219,34 @@ const columnWidthFunction =
: cellSizeCache[columns[index]] || defaultColumnWidth;
};
+const GEOMETRY_FIELD_TYPES: Record<string, boolean> = {
+ geojson: true,
+ geoarrow: true
+};
+const MAX_GEOMETRY_CELL_CHARS = 512;
+const MAX_GEOMETRY_ARRAY_LENGTH = 8;
+
+function formatGeometryForTable(value: unknown): string {
+ if (typeof value === 'string') {
+ return value.length > MAX_GEOMETRY_CELL_CHARS
+ ? `${value.slice(0, MAX_GEOMETRY_CELL_CHARS)}…`
+ : value;
+ }
+ try {
+ const text = JSON.stringify(value, (_key, val) =>
+ Array.isArray(val) && val.length > MAX_GEOMETRY_ARRAY_LENGTH ? `[…${val.length}]` : val
+ );
+ if (!text) {
+ return '';
+ }
+ return text.length > MAX_GEOMETRY_CELL_CHARS
+ ? `${text.slice(0, MAX_GEOMETRY_CELL_CHARS)}…`
+ : text;
+ } catch {
+ return '{geojson}';
+ }
+}
+
interface GetRowCellProps {
dataContainer: DataContainerInterface | null;
columns: (string & {ghost?: boolean})[];
@@ -213,19 +267,16 @@ const defaultGetRowCell = (
const {type} = colMeta[column];
const value = dataContainer?.valueAt(rowIdx, columns.indexOf(column));
- return value === null || value === undefined || value === ''
- ? ''
- : formatter
- ? formatter(value)
- : parseFieldValue(value, type);
-};
-
-type StatsControlProps = {
- top: number;
- showStats?: boolean;
+ if (value === null || value === undefined || value === '') {
+ return '';
+ }
+ if (GEOMETRY_FIELD_TYPES[type]) {
+ return formatGeometryForTable(value);
+ }
+ return formatter ? formatter(value) : parseFieldValue(value, type);
};
-const StyledStatsControl = styled.div<StatsControlProps>`
+const StyledStatsControl = styled.div<{$showStats?: boolean; top: number}>`
height: ${props => props.theme.headerStatsControlHeight}px;
width: 100%;
display: flex;
@@ -233,8 +284,8 @@ const StyledStatsControl = styled.div<StatsControlProps>`
align-items: stretch;
position: absolute;
top: ${props => props.top}px;
- font-family: ${props => props.theme.fontFamilyMedium}px;
- font-size: 12px;
+ font-family: ${props => props.theme.fontFamily};
+ font-size: ${props => props.theme.cellFontSize}px;
color: ${props => props.theme.activeColor};
background-color: ${props => props.theme.headerCellStatsControlBackground};
&:hover {
@@ -248,8 +299,8 @@ const StyledStatsControl = styled.div<StatsControlProps>`
svg {
margin-left: 12px;
- transition: transform 0.5s ease;
- transform: rotate(${props => (props.showStats ? 180 : 0)}deg);
+ transition: transform ${STATS_TOGGLE_DURATION} ease-in-out;
+ transform: rotate(${props => (props.$showStats ? 180 : 0)}deg);
}
}
`;
@@ -263,8 +314,13 @@ const StatsControl = ({
showStats?: boolean;
toggleShowStats: () => void;
}) => (
- <StyledStatsControl top={top} showStats={showStats}>
- <div onClick={toggleShowStats}>
+ <StyledStatsControl
+ top={top}
+ $showStats={showStats}
+ onClick={toggleShowStats}
+ onDoubleClick={event => event.stopPropagation()}
+ >
+ <div>
{showStats ? 'Hide Column Stats' : 'Show Column Stats'}
<ArrowDown height="18px" />
</div>
@@ -292,6 +348,8 @@ interface TableSectionProps {
headerCellRender?;
dataCellRender?;
scrollLeft?: number;
+ headerVisibleHeight?: number;
+ headerBodyOffset?: number;
}
export const TableSection = ({
@@ -308,21 +366,28 @@ export const TableSection = ({
setGridRef = undefined,
headerCellRender,
dataCellRender,
- scrollLeft = 0
+ scrollLeft = 0,
+ headerVisibleHeight,
+ headerBodyOffset
}: TableSectionProps) => {
- const headerHeight = headerGridProps.height;
+ const headerGridHeight = headerGridProps.height;
+ const visibleHeight = headerVisibleHeight ?? headerGridHeight;
+ const bodyOffset = headerBodyOffset ?? visibleHeight;
const headerStyle = useMemo(
() => ({
- height: `${headerHeight}px`
+ height: `${visibleHeight}px`,
+ overflow: 'hidden' as const,
+ transition: `height ${STATS_TOGGLE_DURATION} ease-in-out`
}),
- [headerHeight]
+ [visibleHeight]
);
const contentStyle = useMemo(
() => ({
- top: `${headerHeight}px`
+ top: `${visibleHeight}px`,
+ transition: `top ${STATS_TOGGLE_DURATION} ease-in-out`
}),
- [headerHeight]
+ [visibleHeight]
);
return (
@@ -346,7 +411,7 @@ export const TableSection = ({
cellRenderer={headerCellRender}
{...headerGridProps}
{...gridDimension}
- height={headerGridProps.height + browserScrollBarWidth}
+ height={headerGridHeight + browserScrollBarWidth}
width={headerGridWidth}
scrollLeft={scrollLeft}
onScroll={args => onScroll?.({...args, scrollTop: scrollTop ?? 0})}
@@ -361,7 +426,7 @@ export const TableSection = ({
{...dataGridProps}
{...gridDimension}
className={isPinned ? 'pinned-grid' : 'body-grid'}
- height={dataGridHeight - headerGridProps.height}
+ height={dataGridHeight - bodyOffset}
onScroll={onScroll}
scrollLeft={scrollLeft}
scrollTop={scrollTop}
@@ -395,6 +460,7 @@ export interface DataTableProps {
showStats?: boolean;
hasCustomScrollBarStyle?: boolean;
getRowCell?: (renderDataCellProps: GetRowCellProps, formatter: any) => string | number;
+ loadColumnStats?: (dataId: string, fieldName: string) => void;
}
interface DataTableState {
@@ -437,7 +503,7 @@ function DataTableFactory(
componentDidMount() {
this.hasMounted = true;
window.addEventListener('resize', this.scaleCellsToWidth);
- this.scaleCellsToWidth();
+ this.doScaleCellsToWidth();
}
componentDidUpdate(prevProps) {
@@ -445,7 +511,7 @@ function DataTableFactory(
this.props.cellSizeCache !== prevProps.cellSizeCache ||
this.props.pinnedColumns !== prevProps.pinnedColumns
) {
- this.scaleCellsToWidth();
+ this.doScaleCellsToWidth();
}
}
@@ -493,7 +559,11 @@ function DataTableFactory(
};
doScaleCellsToWidth = () => {
- if (this.hasMounted) this.setState(this.getCellSizeCache());
+ if (!this.hasMounted) return;
+ const propsCache = this.props.cellSizeCache || {};
+ // Column widths are measured after first paint; skip until they exist.
+ if (!Object.keys(propsCache).length) return;
+ this.setState(this.getCellSizeCache());
};
scaleCellsToWidth = debounce(this.doScaleCellsToWidth, 300);
@@ -587,20 +657,18 @@ function DataTableFactory(
rowHeight = defaultRowHeight
} = theme;
+ const collapsedHeaderHeight = hasStats
+ ? headerRowHeight + headerStatsControlHeight
+ : headerRowHeight;
+ const expandedHeaderHeight = hasStats ? headerRowWStatsHeight : headerRowHeight;
+ const visibleHeaderHeight = showStats ? expandedHeaderHeight : collapsedHeaderHeight;
+
const headerGridProps = {
cellSizeCache,
className: 'header-grid',
- height: !hasStats
- ? headerRowHeight
- : showStats
- ? headerRowWStatsHeight
- : headerRowHeight + headerStatsControlHeight,
+ height: expandedHeaderHeight,
rowCount: 1,
- rowHeight: !hasStats
- ? headerRowHeight
- : showStats
- ? headerRowWStatsHeight
- : headerRowHeight + headerStatsControlHeight
+ rowHeight: expandedHeaderHeight
};
const dataGridProps = {
@@ -616,6 +684,7 @@ function DataTableFactory(
className="data-table-container"
ref={this.root}
$hasCustomScrollBarStyle={hasCustomScrollBarStyle}
+ $hasStats={Boolean(hasStats)}
>
{Object.keys(cellSizeCache).length ? (
<>
@@ -630,6 +699,8 @@ function DataTableFactory(
isPinned
columns={pinnedColumns}
headerGridProps={headerGridProps}
+ headerVisibleHeight={visibleHeaderHeight}
+ headerBodyOffset={visibleHeaderHeight}
fixedWidth={pinnedColumnsWidth}
onScroll={args => onScroll({...args, scrollLeft})}
scrollTop={scrollTop}
@@ -660,6 +731,8 @@ function DataTableFactory(
isPinned={false}
columns={unpinnedColumnsGhost}
headerGridProps={headerGridProps}
+ headerVisibleHeight={visibleHeaderHeight}
+ headerBodyOffset={visibleHeaderHeight}
fixedWidth={fixedWidth}
fixedHeight={fixedHeight}
onScroll={onScroll}
@@ -698,7 +771,11 @@ function DataTableFactory(
/>
) : null}
</>
- ) : null}
+ ) : (
+ <StyledTableLoading role="status" aria-label="Loading data table">
+ <LoadingSpinner size={48} />
+ </StyledTableLoading>
+ )}
</Container>
);
}
diff --git a/src/components/src/common/histogram-plot.tsx b/src/components/src/common/histogram-plot.tsx
index e9fde255f..0829bd3cc 100644
--- a/src/components/src/common/histogram-plot.tsx
+++ b/src/components/src/common/histogram-plot.tsx
@@ -77,7 +77,7 @@ export type HistogramMaskModeType = {
const undefinedToZero = (v: number | undefined) => (v ? v : 0);
-interface HistogramPlotProps {
+export interface HistogramPlotProps {
width: number;
height: number;
margin: {top: number; bottom: number; left: number; right: number};
diff --git a/src/components/src/index.ts b/src/components/src/index.ts
index 035d13d56..a44f8429c 100644
--- a/src/components/src/index.ts
+++ b/src/components/src/index.ts
@@ -224,6 +224,7 @@ export {default as ExportVideoControlFactory} from './common/animation-control/e
export {default as WindowActionControlFactory} from './common/animation-control/window-action-control';
export {default as HistogramPlotFactory, HISTOGRAM_MASK_MODE} from './common/histogram-plot';
+export type {HistogramPlotProps} from './common/histogram-plot';
export {default as IconButton} from './common/icon-button';
export {default as LinkRenderer} from './common/link-renderer';
export {
@@ -289,7 +290,13 @@ export {
NumberFormatConfig
} from './common/data-table/display-format';
export {default as HeaderCellFactory} from './common/data-table/header-cell';
+export type {HeaderCellProps} from './common/data-table/header-cell';
export {FormatterDropdown, default as OptionDropdown} from './common/data-table/option-dropdown';
+export {default as ColumnStatisticsNumericFactory} from './common/data-table/column-statistics-numeric';
+export type {ColumnStatisticsNumericProps} from './common/data-table/column-statistics-numeric';
+export {default as ColumnStatisticsTimeFactory} from './common/data-table/column-statistics-time';
+export type {ColumnStatisticsTimeProps} from './common/data-table/column-statistics-time';
+export {STATS_WIDTH} from './common/data-table/column-statistics-components';
export {default as DatasetLabel} from './common/dataset-label';
export {default as FieldSelectorFactory} from './common/field-selector';
export * from './common/field-token';
diff --git a/src/components/src/modal-container.tsx b/src/components/src/modal-container.tsx
index 5b7c5af56..9cb6690e5 100644
--- a/src/components/src/modal-container.tsx
+++ b/src/components/src/modal-container.tsx
@@ -325,6 +325,7 @@ export default function ModalContainerFactory(
pinTableColumn={visStateActions.pinTableColumn}
copyTableColumn={visStateActions.copyTableColumn}
setColumnDisplayFormat={visStateActions.setColumnDisplayFormat}
+ loadColumnStats={visStateActions.loadColumnStats}
uiStateActions={uiStateActions}
uiState={uiState}
/>
diff --git a/src/components/src/modals/data-table-modal.tsx b/src/components/src/modals/data-table-modal.tsx
index e0cf39fb2..a68fb3304 100644
--- a/src/components/src/modals/data-table-modal.tsx
+++ b/src/components/src/modals/data-table-modal.tsx
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import React, {useState, useRef, useMemo, useCallback} from 'react';
+import React, {useState, useRef, useMemo, useCallback, useEffect} from 'react';
import styled, {withTheme, IStyledComponent} from 'styled-components';
import DatasetLabel from '../common/dataset-label';
import DataTableFactory from '../common/data-table';
@@ -10,12 +10,13 @@ import CanvasHack from '../common/data-table/canvas';
import KeplerTable, {Datasets} from '@kepler.gl/table';
import {UIStateActions} from '@kepler.gl/actions';
import {UiState} from '@kepler.gl/types';
+import {getApplicationConfig} from '@kepler.gl/utils';
import {Gear} from '../common/icons';
import Portaled from '../common/portaled';
import DataTableConfigFactory from '../common/data-table/display-format';
import {BaseComponentProps} from '../types';
+import {STATS_WIDTH} from '../common/data-table/column-statistics-components';
-const MIN_STATS_CELL_SIZE = 122;
const DEFAULT_SORT_COLUMN = {};
// sidePadding changes from 38 to 68, 30px for configuration button
@@ -137,6 +138,7 @@ interface DataTableModalProps {
) => void;
uiStateActions: typeof UIStateActions;
uiState: UiState;
+ loadColumnStats?: (dataId: string, fieldName: string | string[]) => void;
}
function DataTableModalFactory(
@@ -154,11 +156,14 @@ function DataTableModalFactory(
showTab = true,
setColumnDisplayFormat: setColumnDisplayFormatProp,
uiStateActions,
- uiState
+ uiState,
+ loadColumnStats
}) => {
const [showConfig, setShowConfig] = useState(false);
const datasetCellSizeCache = useRef<Record<string, any>>({});
+ const enableColumnStats = getApplicationConfig().enableColumnStats;
+
const fields = useMemo(
() => (datasets && dataId ? (datasets[dataId] || {}).fields : undefined),
[datasets, dataId]
@@ -169,14 +174,15 @@ function DataTableModalFactory(
const colMeta = useMemo(
() =>
fields?.reduce(
- (acc, {name, displayName, type, filterProps, format, displayFormat}) => ({
+ (acc, {name, displayName, type, filterProps, format, displayFormat, isLoadingStats}) => ({
...acc,
[name]: {
name: displayName || name,
type,
...(format ? {format} : {}),
...(displayFormat ? {displayFormat} : {}),
- ...(filterProps?.columnStats ? {columnStats: filterProps.columnStats} : {})
+ ...(filterProps?.columnStats ? {columnStats: filterProps.columnStats} : {}),
+ ...(typeof isLoadingStats === 'boolean' ? {isLoadingStats} : {})
}
}),
{}
@@ -184,51 +190,58 @@ function DataTableModalFactory(
[fields]
);
- const cellSizeCache = useMemo(() => {
+ const [cellSizeCache, setCellSizeCache] = useState<
+ Record<string, {row: number; header: number}>
+ >({});
+
+ useEffect(() => {
if (!datasets || !dataId || !datasets[dataId]) {
- return {};
+ setCellSizeCache({});
+ return;
}
- const {fields, dataContainer} = datasets[dataId];
- let showCalculate: boolean | null = null;
- if (!datasetCellSizeCache.current[dataId]) {
- showCalculate = true;
- } else if (
- datasetCellSizeCache.current[dataId].fields !== fields ||
- datasetCellSizeCache.current[dataId].dataContainer !== dataContainer
+ const {fields, dataContainer} = datasets[dataId];
+ const cached = datasetCellSizeCache.current[dataId];
+ if (
+ cached &&
+ cached.fields === fields &&
+ cached.dataContainer === dataContainer &&
+ cached.enableColumnStats === enableColumnStats
) {
- showCalculate = true;
- }
-
- if (!showCalculate) {
- return datasetCellSizeCache.current[dataId].cellSizeCache;
+ setCellSizeCache(cached.cellSizeCache);
+ return;
}
- const cellSizeCache = fields.reduce(
- (acc, field, colIdx) => ({
+ const nextCache = fields.reduce((acc, field, colIdx) => {
+ const size = renderedSize({
+ text: {
+ dataContainer,
+ column: field.displayName
+ },
+ colIdx,
+ type: field.type,
+ fontSize: theme.cellFontSize,
+ font: theme.fontFamily
+ });
+ return {
...acc,
- [field.name]: renderedSize({
- text: {
- dataContainer,
- column: field.displayName
- },
- colIdx,
- type: field.type,
- fontSize: theme.cellFontSize,
- font: theme.fontFamily,
- minCellSize: MIN_STATS_CELL_SIZE
- })
- }),
- {}
- );
+ [field.name]: enableColumnStats
+ ? {
+ row: Math.max(size.row, STATS_WIDTH),
+ header: Math.max(size.header, STATS_WIDTH)
+ }
+ : size
+ };
+ }, {});
datasetCellSizeCache.current[dataId] = {
- cellSizeCache,
+ cellSizeCache: nextCache,
fields,
- dataContainer
+ dataContainer,
+ enableColumnStats
};
- return cellSizeCache;
- }, [dataId, datasets, theme]);
+ setCellSizeCache(nextCache);
+ }, [dataId, datasets, theme, enableColumnStats]);
const handleCopyTableColumn = useCallback(
(column: string) => {
@@ -309,7 +322,8 @@ function DataTableModalFactory(
pinTableColumn={handlePinTableColumn}
sortTableColumn={handleSortTableColumn}
setColumnDisplayFormat={handleSetColumnDisplayFormat}
- hasStats={false}
+ hasStats={enableColumnStats}
+ loadColumnStats={enableColumnStats ? loadColumnStats : undefined}
/>
) : null}
</TableContainer>
diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts
index 6d62d7a38..2df1f70b1 100644
--- a/src/reducers/src/vis-state-updaters.ts
+++ b/src/reducers/src/vis-state-updaters.ts
@@ -45,7 +45,9 @@ import {
fitBounds as fitMapBounds,
toggleLayerForMap,
applyFilterConfig,
- SetLoadingIndicatorPayload
+ SetLoadingIndicatorPayload,
+ loadColumnStatsSuccess,
+ loadColumnStatsError
} from '@kepler.gl/actions';
// Utils
@@ -152,6 +154,7 @@ import {Loader} from '@loaders.gl/loader-utils';
import {
Datasets,
+ FilterProps,
assignGpuChannel,
copyTableAndUpdate,
createNewDataEntry,
@@ -188,7 +191,9 @@ import {
LayerToFilterTimeInterval,
TIME_INTERVALS_ORDERED,
mergeFilterDomain,
- initCustomPaletteByCustomScale
+ initCustomPaletteByCustomScale,
+ collectColumnValues,
+ getColumnStatistics
} from '@kepler.gl/utils';
import {createEffect} from '@kepler.gl/effects';
import {PayloadAction} from '@reduxjs/toolkit';
@@ -4130,6 +4135,123 @@ export function setColumnDisplayFormatUpdater(
return newState;
}
+type LoadColumnStatsTaskPayload = {
+ dataset: VisState['datasets'][string];
+ field: Field;
+};
+
+function getMappedValue(filterProps?: FilterProps | null): unknown[] | undefined {
+ if (filterProps && 'mappedValue' in filterProps && Array.isArray(filterProps.mappedValue)) {
+ return filterProps.mappedValue;
+ }
+ return undefined;
+}
+
+function yieldForPaint(): Promise<void> {
+ return new Promise(resolve => {
+ setTimeout(resolve, 0);
+ });
+}
+
+async function requestDatasetColumnStatistics({dataset, field}: LoadColumnStatsTaskPayload) {
+ // Yield so the data table and per-column loading spinners can paint first.
+ await yieldForPaint();
+ const filterProps = field.filterProps || dataset.getColumnFilterProps(field.name);
+ const mappedValue = getMappedValue(filterProps);
+ const values = mappedValue ?? collectColumnValues(dataset.dataContainer, field.fieldIdx);
+ const result = await getColumnStatistics({
+ values,
+ fieldType: field.type
+ });
+
+ return {result, filterProps};
+}
+
+const LOAD_COLUMN_STATS_TASK = Task.fromPromise(
+ requestDatasetColumnStatistics,
+ 'LOAD_COLUMN_STATS_TASK'
+);
+
+/**
+ * Start loading column statistics for one or more fields.
+ * @memberof visStateUpdaters
+ * @public
+ */
+export function loadColumnStatsUpdater(
+ state: VisState,
+ {dataId, fieldName}: VisStateActions.LoadColumnStatsUpdaterAction
+): VisState {
+ const fieldNamesToLoad = toArray(fieldName);
+ const dataset = state.datasets[dataId];
+ const fieldsToLoad = dataset?.fields.filter(f => fieldNamesToLoad.includes(f.name));
+ if (!dataset || !fieldsToLoad.length) {
+ return state;
+ }
+
+ const newFields = dataset.fields.map(field =>
+ fieldNamesToLoad.includes(field.name) ? {...field, isLoadingStats: true} : field
+ );
+ const newDataset = copyTableAndUpdate(dataset, {fields: newFields});
+
+ const tasks = fieldsToLoad.map(f =>
+ LOAD_COLUMN_STATS_TASK({dataset: newDataset, field: f}).bimap(
+ ({result, filterProps}) => loadColumnStatsSuccess(newDataset.id, f.name, result, filterProps),
+ (error: Error) => loadColumnStatsError(newDataset.id, f.name, error)
+ )
+ );
+ const nextState = pick_('datasets')(merge_({[dataId]: newDataset}))(state);
+
+ return withTask(nextState, tasks);
+}
+
+/**
+ * Persist loaded column statistics on the field.
+ * @memberof visStateUpdaters
+ * @public
+ */
+export function loadColumnStatsSuccessUpdater(
+ state: VisState,
+ {result, filterProps, dataId, fieldName}: VisStateActions.LoadColumnStatsSuccessUpdaterAction
+): VisState {
+ const dataset = state.datasets[dataId];
+ const field = (dataset?.fields || []).find(f => f.name === fieldName);
+ if (!dataset || !field) {
+ return state;
+ }
+
+ const newFilterProps = {
+ ...filterProps,
+ columnStats: result
+ };
+ const newFields = dataset.fields.map(f =>
+ f.name === fieldName ? {...f, filterProps: newFilterProps, isLoadingStats: false} : f
+ );
+ const newDataset = copyTableAndUpdate(dataset, {fields: newFields});
+ return pick_('datasets')(merge_({[dataId]: newDataset}))(state);
+}
+
+/**
+ * Clear loading flag when column statistics fail.
+ * @memberof visStateUpdaters
+ * @public
+ */
+export function loadColumnStatsErrorUpdater(
+ state: VisState,
+ {error, dataId, fieldName}: VisStateActions.LoadColumnStatsErrorUpdaterAction
+): VisState {
+ Console.error(error);
+ const dataset = state.datasets[dataId];
+ const field = (dataset?.fields || []).find(f => f.name === fieldName);
+ if (!dataset || !field) {
+ return state;
+ }
+ const newFields = dataset.fields.map(f =>
+ f.name === fieldName ? {...f, isLoadingStats: false} : f
+ );
+ const newDataset = copyTableAndUpdate(dataset, {fields: newFields});
+ return pick_('datasets')(merge_({[dataId]: newDataset}))(state);
+}
+
/**
* Update editor
*/
diff --git a/src/reducers/src/vis-state.ts b/src/reducers/src/vis-state.ts
index 5e99082fd..88816d2f6 100644
--- a/src/reducers/src/vis-state.ts
+++ b/src/reducers/src/vis-state.ts
@@ -158,6 +158,12 @@ const actionHandler = {
[ActionTypes.SET_COLUMN_DISPLAY_FORMAT]: visStateUpdaters.setColumnDisplayFormatUpdater,
+ [ActionTypes.LOAD_COLUMN_STATS]: visStateUpdaters.loadColumnStatsUpdater,
+
+ [ActionTypes.LOAD_COLUMN_STATS_SUCCESS]: visStateUpdaters.loadColumnStatsSuccessUpdater,
+
+ [ActionTypes.LOAD_COLUMN_STATS_ERROR]: visStateUpdaters.loadColumnStatsErrorUpdater,
+
[ActionTypes.NEXT_FILE_BATCH]: visStateUpdaters.nextFileBatchUpdater,
[ActionTypes.PROCESS_FILE_CONTENT]: visStateUpdaters.processFileContentUpdater,
diff --git a/src/styles/src/base.ts b/src/styles/src/base.ts
index 884102b27..2cb275a01 100644
--- a/src/styles/src/base.ts
+++ b/src/styles/src/base.ts
@@ -386,7 +386,7 @@ export const notificationPanelItemWidth = notificationPanelWidth - 60;
export const notificationPanelItemHeight = 60;
// Data Table
-const headerRowHeight = 70;
+const headerRowHeight = 40;
const headerStatsControlHeight = 32;
const headerRowWStatsHeight = 364;
const rowHeight = 32;
@@ -1093,36 +1093,26 @@ const scrollBar = css`
export const modalScrollBar = css`
&::-webkit-scrollbar {
width: 14px;
- height: 16px;
+ height: 14px;
}
&::-webkit-scrollbar-track {
background: white;
}
- &::-webkit-scrollbar-track:horizontal {
- background: ${props => props.theme.textColorHl};
- }
+
&::-webkit-scrollbar-thumb {
background: ${props => props.theme.scrollbarThumbColorLT};
border: 4px solid white;
+ border-radius: 7px;
}
&::-webkit-scrollbar-corner {
- background: ${props => props.theme.textColorHl};
+ background: white;
}
&::-webkit-scrollbar-thumb:hover {
background: ${props => props.theme.scrollbarThumbColorHoverLT};
}
-
- &::-webkit-scrollbar-thumb:vertical {
- border-radius: 7px;
- }
-
- &::-webkit-scrollbar-thumb:horizontal {
- border-radius: 9px;
- border: 4px solid ${props => props.theme.textColorHl};
- }
`;
export const theme = {
diff --git a/src/types/components.d.ts b/src/types/components.d.ts
index 734541270..4745c91b3 100644
--- a/src/types/components.d.ts
+++ b/src/types/components.d.ts
@@ -26,6 +26,7 @@ export type ColMetaProps = {
format?: string;
columnStats?: any;
displayFormat?: string;
+ isLoadingStats?: boolean;
};
export type ColMeta = {
diff --git a/src/utils/src/application-config.ts b/src/utils/src/application-config.ts
index 5bd5d0eba..047383098 100644
--- a/src/utils/src/application-config.ts
+++ b/src/utils/src/application-config.ts
@@ -177,6 +177,9 @@ export type KeplerApplicationConfig = {
/** Whether to show a map control to toggle between light and dark UI themes. Disabled by default. */
enableThemeToggle?: boolean;
+ /** Whether to show column statistics in the data table modal. Enabled by default. */
+ enableColumnStats?: boolean;
+
/**
* Custom SVG icons to be made available in the icon layer.
* These icons will be merged with the default icons fetched from CDN.
@@ -302,11 +305,13 @@ const DEFAULT_APPLICATION_CONFIG: Required<KeplerApplicationConfig> = {
enableThemeToggle: false,
+ enableColumnStats: true,
+
customIcons: [],
customIconUrl: '',
- reduxLogLevel: 1,
+ reduxLogLevel: 1
};
const applicationConfig: Required<KeplerApplicationConfig> = DEFAULT_APPLICATION_CONFIG;
diff --git a/src/utils/src/column-statistics.ts b/src/utils/src/column-statistics.ts
new file mode 100644
index 000000000..e6a8995a5
--- /dev/null
+++ b/src/utils/src/column-statistics.ts
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import {deviation, mean, quantileSorted} from 'd3-array';
+
+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
+import {Bin} from '@kepler.gl/types';
+
+import {isNumber} from './data-utils';
+import {histogramFromValues} from './plot';
+import quickInsertionSort from './quick-insertion-sort';
+import {DataContainerInterface} from './data-container-interface';
+
+export const NUMERIC_BIN_COUNT = 20;
+
+export enum ColumnStatsType {
+ numeric = 'numeric',
+ categorical = 'categorical',
+ boolean = 'boolean',
+ time = 'time',
+ geo = 'geo',
+ other = 'other'
+}
+
+export type Quantile = {label: string; value: number | null};
+
+export type NumericColumnStats = {
+ type: ColumnStatsType.numeric;
+ bins: Bin[];
+ quantiles: Quantile[];
+ mean: number | null;
+ std: number | null;
+ percentNulls: number | null;
+};
+
+export type CategoricalColumnStats = {
+ type: ColumnStatsType.categorical;
+ uniqueValues: number;
+ percentNulls: number | null;
+};
+
+export type TimeColumnStats = {
+ type: ColumnStatsType.time;
+ bins: Bin[];
+ min: number | null;
+ max: number | null;
+ percentNulls: number | null;
+};
+
+export type BooleanColumnStats = {
+ type: ColumnStatsType.boolean;
+};
+
+export type GeometryColumnStats = {
+ type: ColumnStatsType.geo;
+};
+
+export type UnknownColumnStats = {
+ type: ColumnStatsType.other;
+};
+
+export type ColumnStats =
+ | NumericColumnStats
+ | BooleanColumnStats
+ | CategoricalColumnStats
+ | TimeColumnStats
+ | GeometryColumnStats
+ | UnknownColumnStats;
+
+const QUANTILE_STOPS = [0, 0.25, 0.5, 0.75, 1];
+
+function percentNulls(total: number, valid: number): number | null {
+ return total > 0 ? (total - valid) / total : null;
+}
+
+function getNumericStats(values: readonly unknown[]): NumericColumnStats {
+ const filtered = values.filter(isNumber);
+ quickInsertionSort(filtered);
+
+ const quantiles = QUANTILE_STOPS.map(p => ({
+ label: p === 0 ? 'Min' : p === 1 ? 'Max' : `${p * 100}%`,
+ value: filtered.length ? quantileSorted(filtered, p) ?? null : null
+ }));
+
+ return {
+ type: ColumnStatsType.numeric,
+ bins: histogramFromValues(filtered, NUMERIC_BIN_COUNT),
+ quantiles,
+ percentNulls: percentNulls(values.length, filtered.length),
+ mean: filtered.length ? mean(filtered) ?? null : null,
+ std: filtered.length ? deviation(filtered) ?? null : null
+ };
+}
+
+function isPresentCategoricalValue(v: unknown): boolean {
+ return v !== null && v !== undefined && v !== '';
+}
+
+function getCategoricalStats(values: readonly unknown[]): CategoricalColumnStats {
+ const filtered = values.filter(isPresentCategoricalValue);
+ return {
+ type: ColumnStatsType.categorical,
+ uniqueValues: new Set(filtered).size,
+ percentNulls: percentNulls(values.length, filtered.length)
+ };
+}
+
+function getTimeStats(values: readonly unknown[]): TimeColumnStats {
+ const filtered = values.filter(isNumber);
+ const bins = histogramFromValues(filtered, NUMERIC_BIN_COUNT);
+
+ return {
+ type: ColumnStatsType.time,
+ bins,
+ min: filtered.length ? filtered.reduce((a, b) => (a < b ? a : b)) : null,
+ max: filtered.length ? filtered.reduce((a, b) => (a > b ? a : b)) : null,
+ percentNulls: percentNulls(values.length, filtered.length)
+ };
+}
+
+export function collectColumnValues(
+ dataContainer: DataContainerInterface,
+ fieldIdx: number
+): unknown[] {
+ const n = dataContainer.numRows();
+ const values = new Array(n);
+ for (let i = 0; i < n; i++) {
+ values[i] = dataContainer.valueAt(i, fieldIdx);
+ }
+ return values;
+}
+
+export function computeColumnStats(values: readonly unknown[], fieldType: string): ColumnStats {
+ switch (fieldType) {
+ case ALL_FIELD_TYPES.real:
+ case ALL_FIELD_TYPES.integer:
+ return getNumericStats(values);
+ case ALL_FIELD_TYPES.string:
+ case ALL_FIELD_TYPES.date:
+ case ALL_FIELD_TYPES.boolean:
+ return getCategoricalStats(values);
+ case ALL_FIELD_TYPES.timestamp:
+ return getTimeStats(values);
+ default:
+ return {type: ColumnStatsType.other};
+ }
+}
+
+/**
+ * Compute column statistics asynchronously so the UI can paint a loading state first.
+ */
+export function getColumnStatistics({
+ values,
+ fieldType
+}: {
+ values: readonly unknown[];
+ fieldType: string;
+}): Promise<ColumnStats> {
+ return new Promise(resolve => {
+ resolve(computeColumnStats(values, fieldType));
+ });
+}
diff --git a/src/utils/src/index.ts b/src/utils/src/index.ts
index df90ae725..b38a85654 100644
--- a/src/utils/src/index.ts
+++ b/src/utils/src/index.ts
@@ -33,6 +33,20 @@ export {createNotification, exportImageError, successNotification} from './notif
export {setStyleSheetBaseHref} from './dom-utils';
export {default as domtoimage} from './dom-to-image';
export {getFrequency, getMode, aggregate} from './aggregation';
+export {
+ ColumnStatsType,
+ NUMERIC_BIN_COUNT,
+ collectColumnValues,
+ computeColumnStats,
+ getColumnStatistics
+} from './column-statistics';
+export type {
+ ColumnStats,
+ NumericColumnStats,
+ CategoricalColumnStats,
+ TimeColumnStats,
+ Quantile
+} from './column-statistics';
export {
adjustValueToAnimationWindow,
getBinThresholds,
diff --git a/src/utils/src/plot.ts b/src/utils/src/plot.ts
index d5475c65d..52f549ea0 100644
--- a/src/utils/src/plot.ts
+++ b/src/utils/src/plot.ts
@@ -113,7 +113,7 @@ export function histogramFromValues(
values: (Millisecond | null | number)[],
numBins: number,
valueAccessor?: (d: number) => number
-) {
+): Bin[] {
const getBins = d3Histogram().thresholds(numBins);
if (valueAccessor) {
@@ -131,7 +131,7 @@ export function histogramFromValues(
.filter(b => {
const {x0, x1} = b;
return isNumber(x0) && isNumber(x1);
- });
+ }) as Bin[];
}
export function histogramFromOrdinal(
diff --git a/test/browser/components/modals/data-table-modal-test.js b/test/browser/components/modals/data-table-modal-test.js
index 65bfd645f..0e29af0a7 100644
--- a/test/browser/components/modals/data-table-modal-test.js
+++ b/test/browser/components/modals/data-table-modal-test.js
@@ -32,7 +32,7 @@ import {geoStyleFields, geoStyleRows} from 'test/fixtures/geojson';
import {StateWFiles, testCsvDataId, testGeoJsonDataId} from 'test/helpers/mock-state';
import {STYLED_COMPONENTS_DUPLICATED_ENTRIES} from '../../../helpers/utils';
-import {createDataContainer, getFieldFormatLabels} from '@kepler.gl/utils';
+import {createDataContainer, getFieldFormatLabels, initApplicationConfig} from '@kepler.gl/utils';
const {VertThreeDots} = Icons;
const DataTableModal = appInjector.get(DataTableModalFactory);
@@ -246,6 +246,7 @@ test('Components -> DataTableModal.render: csv 1', t => {
t.deepEqual(props.columns, expectedColumns, 'DataTable should have the correct props.columns');
t.deepEqual(props.colMeta, expectedColMeta, 'DataTable should have the correct props.colMeta');
+ t.equal(props.hasStats, true, 'DataTable should enable column stats by default');
const datasetId = Object.keys(StateWFiles.visState.datasets)[0];
@@ -264,6 +265,20 @@ test('Components -> DataTableModal.render: csv 1', t => {
t.end();
});
+test('Components -> DataTableModal.render: disableColumnStats', t => {
+ initApplicationConfig({enableColumnStats: false});
+ try {
+ const wrapper = mountWithTheme(
+ <DataTableModal datasets={StateWFiles.visState.datasets} dataId={testCsvDataId} />
+ );
+ const props = wrapper.find(DataTable).at(0).props();
+ t.equal(props.hasStats, false, 'DataTable should disable column stats when config flag is off');
+ } finally {
+ initApplicationConfig({enableColumnStats: true});
+ }
+ t.end();
+});
+
test('Components -> DataTableModal -> click tab', t => {
const showDatasetTable = sinon.spy();
@@ -276,7 +291,11 @@ test('Components -> DataTableModal -> click tab', t => {
);
t.equal(wrapper.find(DatasetModalTab).length, 2, 'should render 2 DatasetModalTab');
- t.equal(wrapper.find(DatasetModalTab).at(0).prop('$active'), true, 'prop 0 active should be true');
+ t.equal(
+ wrapper.find(DatasetModalTab).at(0).prop('$active'),
+ true,
+ 'prop 0 active should be true'
+ );
t.equal(
wrapper.find(DatasetModalTab).at(0).find('.dataset-name').text(),
'hello.csv',
@@ -567,47 +586,50 @@ test('Components -> DatableModal -> sort/pin/copy and display format should be c
test('Components -> cellSize -> renderedSize', t => {
prepareMockCanvas();
+ initApplicationConfig({enableColumnStats: true});
+ try {
+ const fields = [
+ {name: testColumns[0], type: 'geojson'},
+ {name: testColumns[1], type: 'real'},
+ {name: testColumns[2], type: 'string'}
+ ];
+ const dataContainer = createDataContainer(texts, {fields});
+
+ const wrapper = mountWithTheme(
+ <DataTableModal
+ datasets={{
+ smoothie: {
+ id: 'smoothie',
+ dataContainer,
+ fields: [
+ {name: testColumns[0], type: 'geojson', displayName: testColumns[0]},
+ {name: testColumns[1], type: 'real', displayName: testColumns[1]},
+ {name: testColumns[2], type: 'string', displayName: testColumns[2]}
+ ],
+ color: [113, 113, 113]
+ }
+ }}
+ dataId="smoothie"
+ />
+ );
- const fields = [
- {name: testColumns[0], type: 'geojson'},
- {name: testColumns[1], type: 'real'},
- {name: testColumns[2], type: 'string'}
- ];
- const dataContainer = createDataContainer(texts, {fields});
-
- const wrapper = mountWithTheme(
- <DataTableModal
- datasets={{
- smoothie: {
- id: 'smoothie',
- dataContainer,
- fields: [
- {name: testColumns[0], type: 'geojson', displayName: testColumns[0]},
- {name: testColumns[1], type: 'real', displayName: testColumns[1]},
- {name: testColumns[2], type: 'string', displayName: testColumns[2]}
- ],
- color: [113, 113, 113]
- }
- }}
- dataId="smoothie"
- />
- );
-
- const props = wrapper.find(DataTable).at(0).props();
-
- const expected = {
- _geojson: {row: 500, header: 186},
- 'income level of people over 65': {row: 162, header: 223},
- engagement: {row: 162, header: 186}
- };
+ const props = wrapper.find(DataTable).at(0).props();
- t.deepEqual(
- props.cellSizeCache,
- expected,
- 'DataTable should have the correct props.cellSizeCache'
- );
+ const expected = {
+ _geojson: {row: 180, header: 180},
+ 'income level of people over 65': {row: 180, header: 223},
+ engagement: {row: 180, header: 180}
+ };
- restoreMockCanvas();
+ t.deepEqual(
+ props.cellSizeCache,
+ expected,
+ 'DataTable should have the correct props.cellSizeCache'
+ );
+ } finally {
+ restoreMockCanvas();
+ initApplicationConfig({enableColumnStats: true});
+ }
t.end();
});
diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.js
index 8c059883e..dc3375a01 100644
--- a/test/node/reducers/vis-state-test.js
+++ b/test/node/reducers/vis-state-test.js
@@ -6230,6 +6230,62 @@ test('#visStateReducer -> SORT_TABLE_COLUMN', t => {
t.end();
});
+test('#visStateReducer -> LOAD_COLUMN_STATS', t => {
+ drainTasksForTesting();
+ const initialState = CloneDeep(StateWFiles.visState);
+
+ const noOpState = reducer(
+ initialState,
+ VisStateActions.loadColumnStats('missing', 'gps_data.lat')
+ );
+ t.equal(noOpState, initialState, 'state should not change when dataset is missing');
+
+ const nextState = reducer(
+ initialState,
+ VisStateActions.loadColumnStats(testCsvDataId, 'gps_data.lat')
+ );
+ const loadingField = nextState.datasets[testCsvDataId].fields.find(f => f.name === 'gps_data.lat');
+ t.equal(loadingField.isLoadingStats, true, 'should set isLoadingStats while loading');
+
+ const tasks = drainTasksForTesting();
+ t.equal(tasks.length, 1, 'should create a column stats task');
+
+ const result = {
+ type: 'numeric',
+ mean: 30,
+ std: 1,
+ percentNulls: 0,
+ bins: [],
+ quantiles: [
+ {label: 'Min', value: 29},
+ {label: 'Max', value: 31}
+ ]
+ };
+ const successState = reducer(
+ nextState,
+ VisStateActions.loadColumnStatsSuccess(testCsvDataId, 'gps_data.lat', result, {})
+ );
+ const successField = successState.datasets[testCsvDataId].fields.find(
+ f => f.name === 'gps_data.lat'
+ );
+ t.equal(successField.isLoadingStats, false, 'should clear isLoadingStats on success');
+ t.deepEqual(
+ successField.filterProps.columnStats,
+ result,
+ 'should store columnStats on filterProps'
+ );
+
+ const errorState = reducer(
+ nextState,
+ VisStateActions.loadColumnStatsError(testCsvDataId, 'gps_data.lat', new Error('stats failed'))
+ );
+ const errorField = errorState.datasets[testCsvDataId].fields.find(f => f.name === 'gps_data.lat');
+ t.equal(errorField.isLoadingStats, false, 'should clear isLoadingStats on error');
+
+ drainTasksForTesting();
+ t.end();
+});
+
test('#visStateReducer -> PIN_TABLE_COLUMN', t => {
const initialState = CloneDeep(StateWFiles.visState);
diff --git a/test/node/utils/column-statistics-test.js b/test/node/utils/column-statistics-test.js
new file mode 100644
index 000000000..9d3528cfe
--- /dev/null
+++ b/test/node/utils/column-statistics-test.js
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import test from 'tape';
+
+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
+import {ColumnStatsType, computeColumnStats, getColumnStatistics} from '@kepler.gl/utils';
+
+test('columnStatistics -> numeric', t => {
+ const stats = computeColumnStats([1, 2, 3, 4, null], ALL_FIELD_TYPES.integer);
+
+ t.equal(stats.type, ColumnStatsType.numeric, 'should return numeric stats');
+ if (stats.type === ColumnStatsType.numeric) {
+ t.equal(stats.mean, 2.5, 'should compute mean from valid numbers');
+ t.equal(stats.percentNulls, 0.2, 'should compute percent nulls');
+ t.equal(stats.quantiles[0].label, 'Min', 'should include min quantile');
+ t.equal(stats.quantiles[0].value, 1, 'min should be 1');
+ t.equal(stats.quantiles[stats.quantiles.length - 1].value, 4, 'max should be 4');
+ t.ok(stats.bins.length > 0, 'should compute histogram bins');
+ }
+ t.end();
+});
+
+test('columnStatistics -> categorical', t => {
+ const stats = computeColumnStats(['a', 'b', 'a', '', null], ALL_FIELD_TYPES.string);
+
+ t.equal(stats.type, ColumnStatsType.categorical, 'should return categorical stats');
+ if (stats.type === ColumnStatsType.categorical) {
+ t.equal(stats.uniqueValues, 2, 'should count unique non-empty strings');
+ t.equal(stats.percentNulls, 0.4, 'should treat empty string and null as nulls');
+ }
+ t.end();
+});
+
+test('columnStatistics -> boolean as categorical', t => {
+ const stats = computeColumnStats([true, false, true, null], ALL_FIELD_TYPES.boolean);
+
+ t.equal(stats.type, ColumnStatsType.categorical, 'should treat boolean as categorical');
+ if (stats.type === ColumnStatsType.categorical) {
+ t.equal(stats.uniqueValues, 2, 'should count true and false as unique values');
+ t.equal(stats.percentNulls, 0.25, 'should compute percent nulls');
+ }
+ t.end();
+});
+
+test('columnStatistics -> timestamp', t => {
+ const stats = computeColumnStats([1000, 3000, 2000], ALL_FIELD_TYPES.timestamp);
+
+ t.equal(stats.type, ColumnStatsType.time, 'should return time stats');
+ if (stats.type === ColumnStatsType.time) {
+ t.equal(stats.min, 1000, 'should compute min timestamp');
+ t.equal(stats.max, 3000, 'should compute max timestamp');
+ t.equal(stats.percentNulls, 0, 'should have no nulls');
+ t.ok(stats.bins.length > 0, 'should compute histogram bins');
+ }
+ t.end();
+});
+
+test('columnStatistics -> other', t => {
+ const stats = computeColumnStats(['{"type":"Point"}'], ALL_FIELD_TYPES.geojson);
+ t.equal(stats.type, ColumnStatsType.other, 'should return other for geojson');
+ t.end();
+});
+
+test('columnStatistics -> getColumnStatistics is async', async t => {
+ const stats = await getColumnStatistics({
+ values: [1, 2, 3],
+ fieldType: ALL_FIELD_TYPES.real
+ });
+ t.equal(stats.type, ColumnStatsType.numeric, 'should resolve numeric stats');
+ t.end();
+});
diff --git a/test/node/utils/index.js b/test/node/utils/index.js
index 0958ec375..724d623c3 100644
--- a/test/node/utils/index.js
+++ b/test/node/utils/index.js
@@ -2,6 +2,7 @@
// Copyright contributors to the kepler.gl project
import './data-utils-test';
+import './column-statistics-test';
import './data-processor-test';
import './kepler-table-test';
import './kepler-table-utils-test';