Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
feat: add column statistics to the data table (#3646)
Igor Dykhta34fd0bfc4999
diff --git a/src/actions/src/action-types.ts b/src/actions/src/action-types.tsindex ab2d20daf..9d1005e53 100644--- a/src/actions/src/action-types.ts+++ b/src/actions/src/action-types.ts@@ -109,6 +109,9 @@ export const ActionTypes = {PIN_TABLE_COLUMN: `${ACTION_PREFIX}PIN_TABLE_COLUMN`,COPY_TABLE_COLUMN: `${ACTION_PREFIX}COPY_TABLE_COLUMN`,SET_COLUMN_DISPLAY_FORMAT: `${ACTION_PREFIX}SET_COLUMN_DISPLAY_FORMAT`,+ LOAD_COLUMN_STATS: `${ACTION_PREFIX}LOAD_COLUMN_STATS`,+ LOAD_COLUMN_STATS_SUCCESS: `${ACTION_PREFIX}LOAD_COLUMN_STATS_SUCCESS`,+ LOAD_COLUMN_STATS_ERROR: `${ACTION_PREFIX}LOAD_COLUMN_STATS_ERROR`,NEXT_FILE_BATCH: `${ACTION_PREFIX}NEXT_FILE_BATCH`,PROCESS_FILE_CONTENT: `${ACTION_PREFIX}PROCESS_FILE_CONTENT`,UPDATE_TABLE_COLOR: `${ACTION_PREFIX}UPDATE_TABLE_COLOR`,diff --git a/src/actions/src/vis-state-actions.ts b/src/actions/src/vis-state-actions.tsindex 1e5a1b4ca..3dd07777b 100644--- a/src/actions/src/vis-state-actions.ts+++ b/src/actions/src/vis-state-actions.ts@@ -1044,6 +1044,83 @@ export function setColumnDisplayFormat(};}+export type LoadColumnStatsUpdaterAction = {+ dataId: string;+ fieldName: string | string[];+};++/**+ * Lazily compute column statistics for the data table header.+ * @param dataId+ * @param fieldName one or more field names+ * @returns action+ * @public+ */+export function loadColumnStats(+ dataId: string,+ fieldName: string | string[]+): Merge<LoadColumnStatsUpdaterAction, {type: typeof ActionTypes.LOAD_COLUMN_STATS}> {+ return {+ type: ActionTypes.LOAD_COLUMN_STATS,+ dataId,+ fieldName+ };+}++export type LoadColumnStatsSuccessUpdaterAction = {+ dataId: string;+ fieldName: string;+ result: any;+ filterProps?: any;+};++/**+ * Column statistics finished loading+ * @returns action+ * @public+ */+export function loadColumnStatsSuccess(+ dataId: string,+ fieldName: string,+ result: any,+ filterProps?: any+): Merge<+ LoadColumnStatsSuccessUpdaterAction,+ {type: typeof ActionTypes.LOAD_COLUMN_STATS_SUCCESS}+> {+ return {+ type: ActionTypes.LOAD_COLUMN_STATS_SUCCESS,+ dataId,+ fieldName,+ result,+ filterProps+ };+}++export type LoadColumnStatsErrorUpdaterAction = {+ dataId: string;+ fieldName: string;+ error: Error;+};++/**+ * Column statistics failed to load+ * @returns action+ * @public+ */+export function loadColumnStatsError(+ dataId: string,+ fieldName: string,+ error: Error+): Merge<LoadColumnStatsErrorUpdaterAction, {type: typeof ActionTypes.LOAD_COLUMN_STATS_ERROR}> {+ return {+ type: ActionTypes.LOAD_COLUMN_STATS_ERROR,+ dataId,+ fieldName,+ error+ };+}+export type AddDataToMapUpdaterOptions = {centerMap?: boolean;padding?: ViewportPadding;diff --git a/src/components/src/common/data-table/cell-size.ts b/src/components/src/common/data-table/cell-size.tsindex 90b0faf0f..3b2686f64 100644--- a/src/components/src/common/data-table/cell-size.ts+++ b/src/components/src/common/data-table/cell-size.ts@@ -13,6 +13,14 @@ const EDGE_COLUMN_PADDING = 10;// be added to min cell width in order to show column nameconst MAX_EMPTY_COLUMN_SPACE = 60;+const GEOMETRY_FIELD_TYPES: Record<string, boolean> = {+ geojson: true,+ geoarrow: true+};++// GeoJSON payloads can be huge; do not size the column from cell text.+const MAX_GEOMETRY_CELL_SIZE = 180;+type RenderSizeParam = {text: {dataContainer: DataContainerInterface; column: string};type?: string;@@ -63,25 +71,32 @@ export function renderedSize({const context = textCanvas.getContext('2d');context.font = [fontSize, font].join('px ');- let rowsToSample = [...Array(numRowsToCalculate)].map(() =>- Math.floor(Math.random() * (dataContainer.numRows() - 1))- );+ const isGeometryColumn = Boolean(GEOMETRY_FIELD_TYPES[type]);+ const maxWidth = isGeometryColumn ? MAX_GEOMETRY_CELL_SIZE : maxCellSize;++ let rowWidth = minCellSize + cellPadding;+ if (!isGeometryColumn) {+ let rowsToSample = [...Array(numRowsToCalculate)].map(() =>+ Math.floor(Math.random() * Math.max(dataContainer.numRows() - 1, 0))+ );- // If we have less than 10 rows, lets measure all of them- if (dataContainer.numRows() <= numRowsToCalculate) {- rowsToSample = Array.from(Array(dataContainer.numRows()).keys());+ // If we have less than 10 rows, lets measure all of them+ if (dataContainer.numRows() <= numRowsToCalculate) {+ rowsToSample = Array.from(Array(dataContainer.numRows()).keys());+ }+ rowWidth = Math.max(+ rowWidth,+ ...rowsToSample.map(rowIdx => {+ const value = parseFieldValue(dataContainer.valueAt(rowIdx, colIdx), type);+ // measuring large text cause slow performance+ if (value.length > maxWidth) {+ return maxWidth;+ }+ const textWidth = context.measureText(value).width;+ return Math.ceil(textWidth) + cellPadding;+ })+ );}- const rowWidth = Math.max(- ...rowsToSample.map(rowIdx => {- const value = parseFieldValue(dataContainer.valueAt(rowIdx, colIdx), type);- // measuring large text cause slow performance- if (value.length > maxCellSize) {- return maxCellSize;- }- const textWidth = context.measureText(value).width;- return Math.ceil(textWidth) + cellPadding;- })- );// header cell only has left paddingconst headerWidth =Math.ceil(context.measureText(column).width) + cellPadding / 2 + optionsButton;@@ -91,7 +106,7 @@ export function renderedSize({// min header width is measured by cellconst minHeaderWidth = minCellSize + cellPadding / 2 + optionsButton;- const clampedRowWidth = clamp(minRowWidth, maxCellSize, rowWidth);+ const clampedRowWidth = clamp(minRowWidth, maxWidth, rowWidth);const clampedHeaderWidth = clamp(minHeaderWidth, maxHeaderSize, headerWidth);// cleanup@@ -154,14 +169,17 @@ function expandCellSize(let remaining = roomToFill;const expandedCellSize = columnOrder.reduce((accu, col) => {- let size = cellSizeCache[col].row;- if (cellSizeCache[col].row < cellSizeCache[col].header && remaining > 0) {+ const colSize = cellSizeCache[col];+ if (!colSize) {+ return accu;+ }++ let size = colSize.row;+ if (colSize.row < colSize.header && remaining > 0) {// if we are cutting off the header, expand to fit itsize =- cellSizeCache[col].header - cellSizeCache[col].row < remaining- ? cellSizeCache[col].header- : cellSizeCache[col].row + remaining;- remaining -= size - cellSizeCache[col].row;+ colSize.header - colSize.row < remaining ? colSize.header : colSize.row + remaining;+ remaining -= size - colSize.row;}return {@@ -172,9 +190,10 @@ function expandCellSize(let ghost: number | null = null;if (remaining > 0 && remaining < MIN_GHOST_CELL_SIZE) {- // expand last cellconst lastCell = columnOrder[columnOrder.length - 1];- expandedCellSize[lastCell] += remaining;+ if (lastCell && expandedCellSize[lastCell] != null) {+ expandedCellSize[lastCell] += remaining;+ }} else if (remaining >= MIN_GHOST_CELL_SIZE) {// if too much left add a ghost cellghost = remaining;diff --git a/src/components/src/common/data-table/column-statistics-categoric.tsx b/src/components/src/common/data-table/column-statistics-categoric.tsxnew file mode 100644index 000000000..a5b711912--- /dev/null+++ b/src/components/src/common/data-table/column-statistics-categoric.tsx@@ -0,0 +1,60 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import React, {useMemo} from 'react';+import styled from 'styled-components';++import {CategoricalColumnStats, isNumber} from '@kepler.gl/utils';++import {+ ColumnStatisticsContainer,+ StatisticRowContainer,+ StatisticRow,+ StyledStatsName,+ TopPlotContainer,+ STATS_MARGIN,+ STATS_WIDTH,+ Margin,+ formatPercent+} from './column-statistics-components';++const StyledHighlightNumber = styled.div`+ font-size: 24px;+ color: ${props => props.theme.activeColorLT};+ width: 100%;+ text-align: center;+`;++export type ColumnStatisticsCategoricalProps = CategoricalColumnStats & {+ width?: number;+ margin?: Margin;+};++const ColumnStatisticsCategorical: React.FC<ColumnStatisticsCategoricalProps> = ({+ width = STATS_WIDTH,+ margin = STATS_MARGIN,+ uniqueValues,+ percentNulls+}) => {+ const nullsRight = useMemo(+ () => (isNumber(percentNulls) ? formatPercent(percentNulls) : null),+ [percentNulls]+ );+ return (+ <ColumnStatisticsContainer width={width} margin={margin}>+ <TopPlotContainer>+ <StatisticRowContainer>+ <StyledStatsName>Unique Values</StyledStatsName>+ </StatisticRowContainer>+ <StyledHighlightNumber>{uniqueValues}</StyledHighlightNumber>+ </TopPlotContainer>+ <StatisticRow+ left="Nulls"+ isBad={isNumber(percentNulls) && percentNulls > 0}+ right={nullsRight}+ />+ </ColumnStatisticsContainer>+ );+};++export default ColumnStatisticsCategorical;diff --git a/src/components/src/common/data-table/column-statistics-components.tsx b/src/components/src/common/data-table/column-statistics-components.tsxnew file mode 100644index 000000000..a09a34b1e--- /dev/null+++ b/src/components/src/common/data-table/column-statistics-components.tsx@@ -0,0 +1,91 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import React from 'react';+import styled from 'styled-components';++export type Margin = {+ top: number;+ left: number;+ right: number;+ bottom: number;+};++export const STATS_WIDTH = 180;+export const STATS_MARGIN: Margin = {top: 10, left: 10, right: 10, bottom: 10};+export const HISTOGRAM_MARGIN: Margin = {top: 10, left: 0, right: 0, bottom: 10};+export const HISTOGRAM_HEIGHT = 50;++type ColumnStatisticsContainerProps = {+ width: number;+ margin: Margin;+};++export const ColumnStatisticsContainer = styled.div.withConfig({+ shouldForwardProp: prop => prop !== 'width' && prop !== 'margin'+})<ColumnStatisticsContainerProps>`+ width: 100%;+ box-sizing: border-box;+ padding-top: ${props => props.margin.top}px;+ padding-bottom: ${props => props.margin.bottom}px;+ padding-left: ${props => props.margin.left}px;+ padding-right: ${props => props.margin.right}px;+ hr {+ border: 0;+ border-top: 1px solid ${props => props.theme.cellBorderColor};+ margin: 8px 0;+ }+`;++export const StatisticRowContainer = styled.div`+ display: flex;+ justify-content: space-between;+ align-items: baseline;+ column-gap: 8px;+ flex-wrap: wrap;+ margin-bottom: 8px;+ font-size: ${props => props.theme.cellFontSize}px;+ line-height: 1.2;+ &:last-child {+ margin-bottom: 0;+ }+`;++type StyledStatsNameProps = {+ $isBad?: boolean;+};++export const StyledStatsName = styled.div<StyledStatsNameProps>`+ flex-shrink: 0;+ color: ${props =>+ props.$isBad ? props.theme.notificationColors?.error || '#f25138' : props.theme.subtextColor};+`;++export const StyledStatsValue = styled.div`+ color: ${props => props.theme.dataTableTextColor};+ white-space: nowrap;+ text-align: right;+ margin-left: auto;+`;++export const TopPlotContainer = styled.div`+ margin-bottom: 12px;+ height: ${HISTOGRAM_HEIGHT + HISTOGRAM_MARGIN.top + HISTOGRAM_MARGIN.bottom}px;+`;++type StatisticRowProps = {+ left: string;+ right: string | null;+ isBad?: boolean;+};++export const StatisticRow: React.FC<StatisticRowProps> = ({left, isBad, right}) => (+ <StatisticRowContainer>+ <StyledStatsName $isBad={isBad}>{left}</StyledStatsName>+ {typeof right === 'string' ? <StyledStatsValue>{right}</StyledStatsValue> : null}+ </StatisticRowContainer>+);++export function formatPercent(n: number): string {+ return `${(n * 100).toFixed(1)}%`;+}diff --git a/src/components/src/common/data-table/column-statistics-numeric.tsx b/src/components/src/common/data-table/column-statistics-numeric.tsxnew file mode 100644index 000000000..6ab446739--- /dev/null+++ b/src/components/src/common/data-table/column-statistics-numeric.tsx@@ -0,0 +1,94 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import React, {useMemo} from 'react';++import {NumericColumnStats, formatNumber, isNumber, roundToFour} from '@kepler.gl/utils';++import HistogramPlotFactory, {HistogramPlotProps} from '../histogram-plot';+import {+ STATS_WIDTH,+ STATS_MARGIN,+ HISTOGRAM_MARGIN,+ HISTOGRAM_HEIGHT,+ TopPlotContainer,+ ColumnStatisticsContainer,+ StatisticRow,+ formatPercent+} from './column-statistics-components';++export type ColumnStatisticsNumericProps = NumericColumnStats & {+ width?: number;+ margin?: {top: number; left: number; right: number; bottom: number};+ type?: string;+};++const VALUE = [-Infinity, Infinity];++ColumnStatisticsNumericFactory.deps = [HistogramPlotFactory];+function ColumnStatisticsNumericFactory(+ HistogramPlot: React.FC<HistogramPlotProps>+): React.FC<ColumnStatisticsNumericProps> {+ const ColumnStatisticsNumeric: React.FC<ColumnStatisticsNumericProps> = ({+ bins,+ quantiles = [],+ mean,+ std,+ percentNulls,+ width = STATS_WIDTH,+ margin = STATS_MARGIN+ }) => {+ const histogramWidth = width - margin.left - margin.right;+ const histogramsByGroup = useMemo(+ () => ({+ bins+ }),+ [bins]+ );+ const nullsRight = useMemo(+ () => (isNumber(percentNulls) ? formatPercent(percentNulls) : null),+ [percentNulls]+ );+ const meanRight = useMemo(() => (isNumber(mean) ? formatNumber(mean, 'real') : null), [mean]);+ const stdRight = useMemo(() => (isNumber(std) ? formatNumber(std, 'real') : null), [std]);+ const quantileDisplays = useMemo(+ () =>+ quantiles.map(row => ({+ label: row.label,+ value: isNumber(row.value)+ ? String(Number.isInteger(row.value) ? row.value : roundToFour(row.value))+ : null+ })),+ [quantiles]+ );++ return (+ <ColumnStatisticsContainer width={width} margin={margin}>+ <TopPlotContainer>+ <HistogramPlot+ histogramsByGroup={histogramsByGroup}+ margin={HISTOGRAM_MARGIN}+ width={histogramWidth}+ height={HISTOGRAM_HEIGHT}+ value={VALUE}+ />+ </TopPlotContainer>+ <StatisticRow+ left="Nulls"+ isBad={isNumber(percentNulls) && percentNulls > 0}+ right={nullsRight}+ />+ <StatisticRow left="Mean" right={meanRight} />+ <StatisticRow left="Std. Dev." right={stdRight} />+ <hr />+ <StatisticRow left="Quantiles" right={null} />+ {quantileDisplays.map((row, index) => (+ <StatisticRow left={row.label} right={row.value} key={index} />+ ))}+ </ColumnStatisticsContainer>+ );+ };++ return ColumnStatisticsNumeric;+}+export default ColumnStatisticsNumericFactory;diff --git a/src/components/src/common/data-table/column-statistics-time.tsx b/src/components/src/common/data-table/column-statistics-time.tsxnew file mode 100644index 000000000..2b72058f6--- /dev/null+++ b/src/components/src/common/data-table/column-statistics-time.tsx@@ -0,0 +1,86 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import React, {useMemo} from 'react';++import {TimeColumnStats, datetimeFormatter, isNumber} from '@kepler.gl/utils';++import HistogramPlotFactory, {HistogramPlotProps} from '../histogram-plot';+import {+ STATS_WIDTH,+ STATS_MARGIN,+ HISTOGRAM_MARGIN,+ HISTOGRAM_HEIGHT,+ StatisticRow,+ ColumnStatisticsContainer,+ TopPlotContainer,+ formatPercent+} from './column-statistics-components';++export type ColumnStatisticsTimeProps = TimeColumnStats & {+ width?: number;+ margin?: {top: number; left: number; right: number; bottom: number};+ type?: string;+ format?: string;+};++const HISTOGRAM_VALUE = [-Infinity, Infinity];++ColumnStatisticsTimeFactory.deps = [HistogramPlotFactory];+function ColumnStatisticsTimeFactory(+ HistogramPlot: React.FC<HistogramPlotProps>+): React.FC<ColumnStatisticsTimeProps> {+ const ColumnStatisticsTime: React.FC<ColumnStatisticsTimeProps> = ({+ bins,+ min,+ max,+ percentNulls,+ width = STATS_WIDTH,+ margin = STATS_MARGIN,+ format+ }) => {+ const histogramWidth = width - margin.left - margin.right;+ const histogramsByGroup = useMemo(+ () => ({+ bins+ }),+ [bins]+ );++ const nullsRight = useMemo(+ () => (isNumber(percentNulls) ? formatPercent(percentNulls) : null),+ [percentNulls]+ );+ const startRight = useMemo(+ () => (isNumber(min) ? datetimeFormatter(null)(format)(min) : null),+ [min, format]+ );+ const endRight = useMemo(+ () => (isNumber(max) ? datetimeFormatter(null)(format)(max) : null),+ [max, format]+ );+ return (+ <ColumnStatisticsContainer width={width} margin={margin}>+ <TopPlotContainer>+ <HistogramPlot+ histogramsByGroup={histogramsByGroup}+ margin={HISTOGRAM_MARGIN}+ width={histogramWidth}+ height={HISTOGRAM_HEIGHT}+ value={HISTOGRAM_VALUE}+ />+ </TopPlotContainer>+ <StatisticRow+ left="Nulls"+ isBad={isNumber(percentNulls) && percentNulls > 0}+ right={nullsRight}+ />+ <StatisticRow left="Start" right={startRight} />+ <StatisticRow left="End" right={endRight} />+ </ColumnStatisticsContainer>+ );+ };++ return ColumnStatisticsTime;+}+export default ColumnStatisticsTimeFactory;diff --git a/src/components/src/common/data-table/header-cell.tsx b/src/components/src/common/data-table/header-cell.tsxindex fca2c8688..82c1e49f7 100644--- a/src/components/src/common/data-table/header-cell.tsx+++ b/src/components/src/common/data-table/header-cell.tsx@@ -1,38 +1,64 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import React, {CSSProperties, useState, useCallback} from 'react';+import React, {CSSProperties, useState, useCallback, useEffect} from 'react';import styled from 'styled-components';import classnames from 'classnames';import Button from './button';import {ArrowUp, ArrowDown, VertThreeDots, Hash} from '../../common/icons';import {SORT_ORDER} from '@kepler.gl/constants';import OptionDropdown, {FormatterDropdown} from './option-dropdown';-import {getFieldFormatLabels} from '@kepler.gl/utils';+import {ColumnStatsType, getFieldFormatLabels} from '@kepler.gl/utils';import {ColMeta} from '@kepler.gl/types';import FieldTokenFactory, {FieldTokenProps} from '../../common/field-token';import {DataTableProps} from './index';+import LoadingSpinner from '../loading-spinner';+import ColumnStatisticsCategorical from './column-statistics-categoric';+import ColumnStatisticsNumericFactory, {+ ColumnStatisticsNumericProps+} from './column-statistics-numeric';+import ColumnStatisticsTimeFactory, {ColumnStatisticsTimeProps} from './column-statistics-time';-const StyledHeaderCell = styled.div`+const StyledHeaderCell = styled.div<{$hasStats?: boolean; $firstCell?: boolean}>`border-bottom: 1px solid ${props => props.theme.headerCellBorderColor};border-top: 1px solid ${props => props.theme.headerCellBorderColor};- padding-top: ${props => props.theme.headerPaddingTop}px;+ border-right: ${props => (props.$hasStats ? `1px solid ${props.theme.cellBorderColor}` : 'none')};+ padding-top: ${props => (props.$hasStats ? 0 : props.theme.headerPaddingTop)}px;padding-right: 0;- padding-bottom: ${props => props.theme.headerPaddingBottom}px;- padding-left: ${props => props.theme.cellPaddingSide}px;- align-items: center;- justify-content: space-between;+ padding-bottom: ${props => (props.$hasStats ? 0 : props.theme.headerPaddingBottom)}px;+ padding-left: ${props => (props.$hasStats ? 0 : props.theme.cellPaddingSide)}px;+ align-items: ${props => (props.$hasStats ? 'stretch' : 'center')};+ justify-content: ${props => (props.$hasStats ? 'flex-start' : 'space-between')};display: flex;- flex-direction: row;+ flex-direction: ${props => (props.$hasStats ? 'column' : 'row')};background-color: ${props => props.theme.headerCellBackground};+ overflow: hidden;.n-sort-idx {font-size: 9px;}+ .header-cell__top {+ display: flex;+ flex-direction: row;+ justify-content: space-between;+ align-items: center;+ width: 100%;+ height: ${props => props.theme.headerRowHeight}px;+ flex: 0 0 auto;+ box-sizing: border-box;+ padding-top: ${props => props.theme.headerPaddingTop}px;+ padding-bottom: ${props => props.theme.headerPaddingBottom}px;+ padding-left: ${props =>+ props.$firstCell+ ? props.theme.cellPaddingSide + props.theme.edgeCellPaddingSide+ : props.theme.cellPaddingSide}px;+ }.details {font-weight: 500;+ font-size: ${props => props.theme.cellFontSize}px;display: flex;- flex-direction: column;+ flex-direction: row;+ align-items: center;justify-content: flex-start;height: 100%;overflow: hidden;@@ -42,6 +68,9 @@ const StyledHeaderCell = styled.div`display: flex;align-items: center;justify-content: space-between;+ margin-left: 6px;+ min-width: 0;+ flex: 1;.col-name__left {display: flex;@@ -71,6 +100,34 @@ const StyledHeaderCell = styled.div`}`;+type StyledBottomStatsProps = {+ $showStats?: boolean;+};++const StyledBottomStats = styled.div<StyledBottomStatsProps>`+ background-color: ${props => props.theme.headerCellStatsBackground};+ flex-grow: 1;+ min-height: 0;+ overflow: hidden;+ opacity: ${props => (props.$showStats ? 1 : 0)};+ transition: opacity 0.5s ease-in-out;+ pointer-events: ${props => (props.$showStats ? 'auto' : 'none')};+ width: 100%;+`;++const StyledLoading = styled.div`+ width: 100%;+ padding-top: 16px;+ display: flex;+ justify-content: center;+`;++const StyledStatsSpacer = styled.div`+ height: ${props => props.theme.headerStatsControlHeight}px;+ width: 100%;+ flex-grow: 0;+`;+type CellInfo = {columnIndex: number;isScrolling: boolean;@@ -81,7 +138,7 @@ type CellInfo = {style: CSSProperties;};-type HeaderCellProps = {+export type HeaderCellProps = {// passed down from react virtualized GridcellInfo: CellInfo;columns: DataTableProps['columns'];@@ -94,11 +151,33 @@ type HeaderCellProps = {style: CSSProperties;};-const HeaderCellFactory = (FieldToken: React.FC<FieldTokenProps>) => {+function HeaderCellFactory(+ FieldToken: React.FC<FieldTokenProps>,+ ColumnStatisticsNumeric: React.FC<ColumnStatisticsNumericProps>,+ ColumnStatisticsTime: React.FC<ColumnStatisticsTimeProps>+) {+ const ColumnStatisticsSection: React.FC<{+ columnStats: any;+ type: string;+ format?: string;+ }> = ({columnStats, type, format}) => {+ switch (columnStats.type) {+ case ColumnStatsType.numeric:+ return <ColumnStatisticsNumeric {...columnStats} type={type} />;+ case ColumnStatsType.categorical:+ return <ColumnStatisticsCategorical {...columnStats} />;+ case ColumnStatsType.time:+ return <ColumnStatisticsTime {...columnStats} format={format} />;+ default:+ return <div />;+ }+ };+const HeaderCell = ({cellInfo,columns,isPinned,+ showStats,props,toggleMoreOptions,moreOptionsColumn@@ -110,34 +189,125 @@ const HeaderCellFactory = (FieldToken: React.FC<FieldTokenProps>) => {sortTableColumn,pinTableColumn,copyTableColumn,- setColumnDisplayFormat+ setColumnDisplayFormat,+ hasStats,+ dataId,+ loadColumnStats} = props;const [showFormatter, setShowFormatter] = useState(false);const column = columns[columnIndex];- const isGhost = column.ghost;- const isSorted = sortColumn[column];+ const isGhost = Boolean(column.ghost);+ const isSorted = isGhost ? false : sortColumn[column];const firstCell = columnIndex === 0;- const isFormatted = Boolean(colMeta[column]?.displayFormat);- const formatLabels = isFormatted ? getFieldFormatLabels(colMeta[column].type) : [];- const onSortTable = useCallback(() => sortTableColumn?.(column), [sortTableColumn, column]);- const onToggleOptionMenu = useCallback(- () => toggleMoreOptions(column),- [toggleMoreOptions, column]- );- const onPin = useCallback(() => pinTableColumn(column), [pinTableColumn, column]);- const onCopy = useCallback(() => copyTableColumn(column), [copyTableColumn, column]);+ const {columnStats, isLoadingStats, type, format} = colMeta[isGhost ? '' : column] || {};+ const isFormatted = isGhost ? false : Boolean(colMeta[column]?.displayFormat);+ const formatLabels = isFormatted && !isGhost ? getFieldFormatLabels(colMeta[column].type) : [];++ useEffect(() => {+ if (+ !isGhost &&+ !columnStats &&+ !isLoadingStats &&+ dataId &&+ showStats &&+ hasStats &&+ loadColumnStats+ ) {+ loadColumnStats(dataId, column);+ }+ }, [+ isGhost,+ columnStats,+ isLoadingStats,+ column,+ dataId,+ showStats,+ hasStats,+ loadColumnStats+ ]);++ const onSortTable = useCallback(() => {+ if (!isGhost) sortTableColumn?.(column);+ }, [sortTableColumn, isGhost, column]);+ const onToggleOptionMenu = useCallback(() => {+ if (!isGhost) toggleMoreOptions(column);+ }, [toggleMoreOptions, isGhost, column]);+ const onPin = useCallback(() => {+ if (!isGhost) pinTableColumn(column);+ }, [pinTableColumn, isGhost, column]);+ const onCopy = useCallback(() => {+ if (!isGhost) copyTableColumn(column);+ }, [copyTableColumn, isGhost, column]);const onSetDisplayFormat = useCallback(displayFormat => {- setColumnDisplayFormat?.({[column]: displayFormat.format});+ if (!isGhost) setColumnDisplayFormat?.({[column]: displayFormat.format});},- [column, setColumnDisplayFormat]+ [column, isGhost, setColumnDisplayFormat]);const onToggleDisplayFormat = useCallback(() => {setShowFormatter(!showFormatter);}, [showFormatter]);+ const headerDetails = isGhost ? (+ <div />+ ) : (+ <>+ <section+ className="details"+ onClick={e => {+ if (e.shiftKey && !isGhost) sortTableColumn?.(column);+ }}+ onDoubleClick={onSortTable}+ >+ <FieldToken type={colMeta[column].type} />+ <div className="col-name">+ <div className="col-name__left">+ <div className="col-name__name">{colMeta[column].name}</div>+ <Button className="col-name__sort" onClick={onSortTable}>+ {isSorted ? (+ isSorted === SORT_ORDER.ASCENDING ? (+ <ArrowUp height="14px" />+ ) : (+ <ArrowDown height="14px" />+ )+ ) : null}+ </Button>+ <Button className="col-name__format" onClick={onToggleDisplayFormat}>+ {isFormatted ? <Hash height="14px" /> : null}+ <FormatterDropdown+ left={0}+ top={0}+ isOpened={isFormatted && showFormatter}+ displayFormat={colMeta[column].displayFormat}+ setDisplayFormat={onSetDisplayFormat}+ onClose={() => setShowFormatter(false)}+ formatLabels={formatLabels}+ />+ </Button>+ </div>+ <Button className="more" onClick={onToggleOptionMenu}>+ <VertThreeDots height="14px" />+ </Button>+ </div>+ </section>++ <section className="options">+ <OptionDropdown+ isOpened={moreOptionsColumn === column}+ column={column}+ colMeta={colMeta}+ toggleMoreOptions={toggleMoreOptions}+ sortTableColumn={sortTableColumn ? mode => sortTableColumn(column, mode) : undefined}+ pinTableColumn={onPin}+ copyTableColumn={onCopy}+ setDisplayFormat={setColumnDisplayFormat ? onSetDisplayFormat : undefined}+ />+ </section>+ </>+ );+return (<StyledHeaderCellclassName={classnames('header-cell', {@@ -147,69 +317,35 @@ const HeaderCellFactory = (FieldToken: React.FC<FieldTokenProps>) => {})}key={key}style={style}- onClick={e => {- e.shiftKey ? sortTableColumn(column) : null;- }}- onDoubleClick={onSortTable}- title={column}+ $hasStats={Boolean(hasStats)}+ $firstCell={firstCell}+ title={isGhost ? undefined : column}>- {isGhost ? (- <div />+ {hasStats && !isGhost ? (+ <div className="header-cell__top">{headerDetails}</div>) : (- <>- <section className="details">- <div className="col-name">- <div className="col-name__left">- <div className="col-name__name">{colMeta[column].name}</div>- <Button className="col-name__sort" onClick={onSortTable}>- {isSorted ? (- isSorted === SORT_ORDER.ASCENDING ? (- <ArrowUp height="14px" />- ) : (- <ArrowDown height="14px" />- )- ) : null}- </Button>- <Button className="col-name__format" onClick={onToggleDisplayFormat}>- {isFormatted ? <Hash height="14px" /> : null}- <FormatterDropdown- left={0}- top={0}- isOpened={isFormatted && showFormatter}- displayFormat={colMeta[column].displayFormat}- setDisplayFormat={onSetDisplayFormat}- onClose={() => setShowFormatter(false)}- formatLabels={formatLabels}- />- </Button>- </div>- <Button className="more" onClick={onToggleOptionMenu}>- <VertThreeDots height="14px" />- </Button>- </div>- <FieldToken type={colMeta[column].type} />- </section>-- <section className="options">- <OptionDropdown- isOpened={moreOptionsColumn === column}- column={column}- colMeta={colMeta}- toggleMoreOptions={toggleMoreOptions}- sortTableColumn={- sortTableColumn ? mode => sortTableColumn(column, mode) : undefined- }- pinTableColumn={onPin}- copyTableColumn={onCopy}- setDisplayFormat={setColumnDisplayFormat ? onSetDisplayFormat : undefined}- />- </section>- </>+ headerDetails)}+ {hasStats && !isGhost ? <StyledStatsSpacer /> : null}+ {hasStats && !isGhost ? (+ <StyledBottomStats className="bottom-stats" $showStats={showStats}>+ {isLoadingStats ? (+ <StyledLoading>+ <LoadingSpinner />+ </StyledLoading>+ ) : columnStats ? (+ <ColumnStatisticsSection columnStats={columnStats} type={type} format={format} />+ ) : null}+ </StyledBottomStats>+ ) : null}</StyledHeaderCell>);};return HeaderCell;-};-HeaderCellFactory.deps = [FieldTokenFactory];+}+HeaderCellFactory.deps = [+ FieldTokenFactory,+ ColumnStatisticsNumericFactory,+ ColumnStatisticsTimeFactory+];export default HeaderCellFactory;diff --git a/src/components/src/common/data-table/index.tsx b/src/components/src/common/data-table/index.tsxindex 46b22b4f8..23a1a39fb 100644--- a/src/components/src/common/data-table/index.tsx+++ b/src/components/src/common/data-table/index.tsx@@ -9,6 +9,7 @@ import {createSelector} from 'reselect';import get from 'lodash/get';import debounce from 'lodash/debounce';import {ArrowDown} from '../icons';+import LoadingSpinner from '../loading-spinner';import {CellSizeCache} from './cell-size';@@ -24,6 +25,7 @@ import {ALL_FIELD_TYPES} from '@kepler.gl/constants';const defaultHeaderRowHeight = 55;const defaultHeaderStatsControlHeight = 40;const defaultRowHeight = 32;+const STATS_TOGGLE_DURATION = '0.5s';const overscanColumnCount = 10;const overscanRowCount = 10;// The default scrollbar width can range anywhere from 12px to 17px@@ -45,11 +47,12 @@ const unpinnedClassList = {type ContainerProps = {$hasCustomScrollBarStyle?: boolean;+ $hasStats?: boolean;};export const Container = styled.div<ContainerProps>`display: flex;- font-size: 11px;+ font-size: ${props => props.theme.cellFontSize}px;flex-grow: 1;color: ${props => props.theme.dataTableTextColor};width: 100%;@@ -134,8 +137,6 @@ export const Container = styled.div<ContainerProps>`height: 100%;display: flex;flex-direction: column;- justify-content: center;- align-items: flex-start;text-align: center;overflow: hidden;// header border is rendered by header container@@ -145,6 +146,8 @@ export const Container = styled.div<ContainerProps>`}}.cell {+ justify-content: center;+ align-items: flex-start;border-bottom: 1px solid ${props => props.theme.cellBorderColor};border-right: 1px solid ${props => props.theme.cellBorderColor};white-space: nowrap;@@ -156,15 +159,27 @@ export const Container = styled.div<ContainerProps>`text-decoration: none;}}- .cell.end-cell,- .header-cell.end-cell {+ .header-cell {+ justify-content: ${props => (props.$hasStats ? 'flex-start' : 'center')};+ align-items: ${props => (props.$hasStats ? 'stretch' : 'flex-start')};+ }+ .cell.end-cell {border-right: none;padding-right: ${props => props.theme.cellPaddingSide + props.theme.edgeCellPaddingSide}px;}- .cell.first-cell,- .header-cell.first-cell {+ .header-cell.end-cell {+ border-right: ${props =>+ props.$hasStats ? `1px solid ${props.theme.cellBorderColor}` : 'none'};+ padding-right: ${props =>+ props.$hasStats ? 0 : props.theme.cellPaddingSide + props.theme.edgeCellPaddingSide}px;+ }+ .cell.first-cell {padding-left: ${props => props.theme.cellPaddingSide + props.theme.edgeCellPaddingSide}px;}+ .header-cell.first-cell {+ padding-left: ${props =>+ props.$hasStats ? 0 : props.theme.cellPaddingSide + props.theme.edgeCellPaddingSide}px;+ }.cell.bottom-cell {border-bottom: none;}@@ -178,6 +193,17 @@ export const Container = styled.div<ContainerProps>`}`;+const StyledTableLoading = styled.div.attrs({+ className: 'data-table-loading'+})`+ display: flex;+ flex-grow: 1;+ align-items: center;+ justify-content: center;+ width: 100%;+ min-height: 240px;+`;+const defaultColumnWidth = 200;export type SortColumn = {@@ -193,6 +219,34 @@ const columnWidthFunction =: cellSizeCache[columns[index]] || defaultColumnWidth;};+const GEOMETRY_FIELD_TYPES: Record<string, boolean> = {+ geojson: true,+ geoarrow: true+};+const MAX_GEOMETRY_CELL_CHARS = 512;+const MAX_GEOMETRY_ARRAY_LENGTH = 8;++function formatGeometryForTable(value: unknown): string {+ if (typeof value === 'string') {+ return value.length > MAX_GEOMETRY_CELL_CHARS+ ? `${value.slice(0, MAX_GEOMETRY_CELL_CHARS)}…`+ : value;+ }+ try {+ const text = JSON.stringify(value, (_key, val) =>+ Array.isArray(val) && val.length > MAX_GEOMETRY_ARRAY_LENGTH ? `[…${val.length}]` : val+ );+ if (!text) {+ return '';+ }+ return text.length > MAX_GEOMETRY_CELL_CHARS+ ? `${text.slice(0, MAX_GEOMETRY_CELL_CHARS)}…`+ : text;+ } catch {+ return '{geojson}';+ }+}+interface GetRowCellProps {dataContainer: DataContainerInterface | null;columns: (string & {ghost?: boolean})[];@@ -213,19 +267,16 @@ const defaultGetRowCell = (const {type} = colMeta[column];const value = dataContainer?.valueAt(rowIdx, columns.indexOf(column));- return value === null || value === undefined || value === ''- ? ''- : formatter- ? formatter(value)- : parseFieldValue(value, type);-};--type StatsControlProps = {- top: number;- showStats?: boolean;+ if (value === null || value === undefined || value === '') {+ return '';+ }+ if (GEOMETRY_FIELD_TYPES[type]) {+ return formatGeometryForTable(value);+ }+ return formatter ? formatter(value) : parseFieldValue(value, type);};-const StyledStatsControl = styled.div<StatsControlProps>`+const StyledStatsControl = styled.div<{$showStats?: boolean; top: number}>`height: ${props => props.theme.headerStatsControlHeight}px;width: 100%;display: flex;@@ -233,8 +284,8 @@ const StyledStatsControl = styled.div<StatsControlProps>`align-items: stretch;position: absolute;top: ${props => props.top}px;- font-family: ${props => props.theme.fontFamilyMedium}px;- font-size: 12px;+ font-family: ${props => props.theme.fontFamily};+ font-size: ${props => props.theme.cellFontSize}px;color: ${props => props.theme.activeColor};background-color: ${props => props.theme.headerCellStatsControlBackground};&:hover {@@ -248,8 +299,8 @@ const StyledStatsControl = styled.div<StatsControlProps>`svg {margin-left: 12px;- transition: transform 0.5s ease;- transform: rotate(${props => (props.showStats ? 180 : 0)}deg);+ transition: transform ${STATS_TOGGLE_DURATION} ease-in-out;+ transform: rotate(${props => (props.$showStats ? 180 : 0)}deg);}}`;@@ -263,8 +314,13 @@ const StatsControl = ({showStats?: boolean;toggleShowStats: () => void;}) => (- <StyledStatsControl top={top} showStats={showStats}>- <div onClick={toggleShowStats}>+ <StyledStatsControl+ top={top}+ $showStats={showStats}+ onClick={toggleShowStats}+ onDoubleClick={event => event.stopPropagation()}+ >+ <div>{showStats ? 'Hide Column Stats' : 'Show Column Stats'}<ArrowDown height="18px" /></div>@@ -292,6 +348,8 @@ interface TableSectionProps {headerCellRender?;dataCellRender?;scrollLeft?: number;+ headerVisibleHeight?: number;+ headerBodyOffset?: number;}export const TableSection = ({@@ -308,21 +366,28 @@ export const TableSection = ({setGridRef = undefined,headerCellRender,dataCellRender,- scrollLeft = 0+ scrollLeft = 0,+ headerVisibleHeight,+ headerBodyOffset}: TableSectionProps) => {- const headerHeight = headerGridProps.height;+ const headerGridHeight = headerGridProps.height;+ const visibleHeight = headerVisibleHeight ?? headerGridHeight;+ const bodyOffset = headerBodyOffset ?? visibleHeight;const headerStyle = useMemo(() => ({- height: `${headerHeight}px`+ height: `${visibleHeight}px`,+ overflow: 'hidden' as const,+ transition: `height ${STATS_TOGGLE_DURATION} ease-in-out`}),- [headerHeight]+ [visibleHeight]);const contentStyle = useMemo(() => ({- top: `${headerHeight}px`+ top: `${visibleHeight}px`,+ transition: `top ${STATS_TOGGLE_DURATION} ease-in-out`}),- [headerHeight]+ [visibleHeight]);return (@@ -346,7 +411,7 @@ export const TableSection = ({cellRenderer={headerCellRender}{...headerGridProps}{...gridDimension}- height={headerGridProps.height + browserScrollBarWidth}+ height={headerGridHeight + browserScrollBarWidth}width={headerGridWidth}scrollLeft={scrollLeft}onScroll={args => onScroll?.({...args, scrollTop: scrollTop ?? 0})}@@ -361,7 +426,7 @@ export const TableSection = ({{...dataGridProps}{...gridDimension}className={isPinned ? 'pinned-grid' : 'body-grid'}- height={dataGridHeight - headerGridProps.height}+ height={dataGridHeight - bodyOffset}onScroll={onScroll}scrollLeft={scrollLeft}scrollTop={scrollTop}@@ -395,6 +460,7 @@ export interface DataTableProps {showStats?: boolean;hasCustomScrollBarStyle?: boolean;getRowCell?: (renderDataCellProps: GetRowCellProps, formatter: any) => string | number;+ loadColumnStats?: (dataId: string, fieldName: string) => void;}interface DataTableState {@@ -437,7 +503,7 @@ function DataTableFactory(componentDidMount() {this.hasMounted = true;window.addEventListener('resize', this.scaleCellsToWidth);- this.scaleCellsToWidth();+ this.doScaleCellsToWidth();}componentDidUpdate(prevProps) {@@ -445,7 +511,7 @@ function DataTableFactory(this.props.cellSizeCache !== prevProps.cellSizeCache ||this.props.pinnedColumns !== prevProps.pinnedColumns) {- this.scaleCellsToWidth();+ this.doScaleCellsToWidth();}}@@ -493,7 +559,11 @@ function DataTableFactory(};doScaleCellsToWidth = () => {- if (this.hasMounted) this.setState(this.getCellSizeCache());+ if (!this.hasMounted) return;+ const propsCache = this.props.cellSizeCache || {};+ // Column widths are measured after first paint; skip until they exist.+ if (!Object.keys(propsCache).length) return;+ this.setState(this.getCellSizeCache());};scaleCellsToWidth = debounce(this.doScaleCellsToWidth, 300);@@ -587,20 +657,18 @@ function DataTableFactory(rowHeight = defaultRowHeight} = theme;+ const collapsedHeaderHeight = hasStats+ ? headerRowHeight + headerStatsControlHeight+ : headerRowHeight;+ const expandedHeaderHeight = hasStats ? headerRowWStatsHeight : headerRowHeight;+ const visibleHeaderHeight = showStats ? expandedHeaderHeight : collapsedHeaderHeight;+const headerGridProps = {cellSizeCache,className: 'header-grid',- height: !hasStats- ? headerRowHeight- : showStats- ? headerRowWStatsHeight- : headerRowHeight + headerStatsControlHeight,+ height: expandedHeaderHeight,rowCount: 1,- rowHeight: !hasStats- ? headerRowHeight- : showStats- ? headerRowWStatsHeight- : headerRowHeight + headerStatsControlHeight+ rowHeight: expandedHeaderHeight};const dataGridProps = {@@ -616,6 +684,7 @@ function DataTableFactory(className="data-table-container"ref={this.root}$hasCustomScrollBarStyle={hasCustomScrollBarStyle}+ $hasStats={Boolean(hasStats)}>{Object.keys(cellSizeCache).length ? (<>@@ -630,6 +699,8 @@ function DataTableFactory(isPinnedcolumns={pinnedColumns}headerGridProps={headerGridProps}+ headerVisibleHeight={visibleHeaderHeight}+ headerBodyOffset={visibleHeaderHeight}fixedWidth={pinnedColumnsWidth}onScroll={args => onScroll({...args, scrollLeft})}scrollTop={scrollTop}@@ -660,6 +731,8 @@ function DataTableFactory(isPinned={false}columns={unpinnedColumnsGhost}headerGridProps={headerGridProps}+ headerVisibleHeight={visibleHeaderHeight}+ headerBodyOffset={visibleHeaderHeight}fixedWidth={fixedWidth}fixedHeight={fixedHeight}onScroll={onScroll}@@ -698,7 +771,11 @@ function DataTableFactory(/>) : null}</>- ) : null}+ ) : (+ <StyledTableLoading role="status" aria-label="Loading data table">+ <LoadingSpinner size={48} />+ </StyledTableLoading>+ )}</Container>);}diff --git a/src/components/src/common/histogram-plot.tsx b/src/components/src/common/histogram-plot.tsxindex e9fde255f..0829bd3cc 100644--- a/src/components/src/common/histogram-plot.tsx+++ b/src/components/src/common/histogram-plot.tsx@@ -77,7 +77,7 @@ export type HistogramMaskModeType = {const undefinedToZero = (v: number | undefined) => (v ? v : 0);-interface HistogramPlotProps {+export interface HistogramPlotProps {width: number;height: number;margin: {top: number; bottom: number; left: number; right: number};diff --git a/src/components/src/index.ts b/src/components/src/index.tsindex 035d13d56..a44f8429c 100644--- a/src/components/src/index.ts+++ b/src/components/src/index.ts@@ -224,6 +224,7 @@ export {default as ExportVideoControlFactory} from './common/animation-control/eexport {default as WindowActionControlFactory} from './common/animation-control/window-action-control';export {default as HistogramPlotFactory, HISTOGRAM_MASK_MODE} from './common/histogram-plot';+export type {HistogramPlotProps} from './common/histogram-plot';export {default as IconButton} from './common/icon-button';export {default as LinkRenderer} from './common/link-renderer';export {@@ -289,7 +290,13 @@ export {NumberFormatConfig} from './common/data-table/display-format';export {default as HeaderCellFactory} from './common/data-table/header-cell';+export type {HeaderCellProps} from './common/data-table/header-cell';export {FormatterDropdown, default as OptionDropdown} from './common/data-table/option-dropdown';+export {default as ColumnStatisticsNumericFactory} from './common/data-table/column-statistics-numeric';+export type {ColumnStatisticsNumericProps} from './common/data-table/column-statistics-numeric';+export {default as ColumnStatisticsTimeFactory} from './common/data-table/column-statistics-time';+export type {ColumnStatisticsTimeProps} from './common/data-table/column-statistics-time';+export {STATS_WIDTH} from './common/data-table/column-statistics-components';export {default as DatasetLabel} from './common/dataset-label';export {default as FieldSelectorFactory} from './common/field-selector';export * from './common/field-token';diff --git a/src/components/src/modal-container.tsx b/src/components/src/modal-container.tsxindex 5b7c5af56..9cb6690e5 100644--- a/src/components/src/modal-container.tsx+++ b/src/components/src/modal-container.tsx@@ -325,6 +325,7 @@ export default function ModalContainerFactory(pinTableColumn={visStateActions.pinTableColumn}copyTableColumn={visStateActions.copyTableColumn}setColumnDisplayFormat={visStateActions.setColumnDisplayFormat}+ loadColumnStats={visStateActions.loadColumnStats}uiStateActions={uiStateActions}uiState={uiState}/>diff --git a/src/components/src/modals/data-table-modal.tsx b/src/components/src/modals/data-table-modal.tsxindex e0cf39fb2..a68fb3304 100644--- a/src/components/src/modals/data-table-modal.tsx+++ b/src/components/src/modals/data-table-modal.tsx@@ -1,7 +1,7 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import React, {useState, useRef, useMemo, useCallback} from 'react';+import React, {useState, useRef, useMemo, useCallback, useEffect} from 'react';import styled, {withTheme, IStyledComponent} from 'styled-components';import DatasetLabel from '../common/dataset-label';import DataTableFactory from '../common/data-table';@@ -10,12 +10,13 @@ import CanvasHack from '../common/data-table/canvas';import KeplerTable, {Datasets} from '@kepler.gl/table';import {UIStateActions} from '@kepler.gl/actions';import {UiState} from '@kepler.gl/types';+import {getApplicationConfig} from '@kepler.gl/utils';import {Gear} from '../common/icons';import Portaled from '../common/portaled';import DataTableConfigFactory from '../common/data-table/display-format';import {BaseComponentProps} from '../types';+import {STATS_WIDTH} from '../common/data-table/column-statistics-components';-const MIN_STATS_CELL_SIZE = 122;const DEFAULT_SORT_COLUMN = {};// sidePadding changes from 38 to 68, 30px for configuration button@@ -137,6 +138,7 @@ interface DataTableModalProps {) => void;uiStateActions: typeof UIStateActions;uiState: UiState;+ loadColumnStats?: (dataId: string, fieldName: string | string[]) => void;}function DataTableModalFactory(@@ -154,11 +156,14 @@ function DataTableModalFactory(showTab = true,setColumnDisplayFormat: setColumnDisplayFormatProp,uiStateActions,- uiState+ uiState,+ loadColumnStats}) => {const [showConfig, setShowConfig] = useState(false);const datasetCellSizeCache = useRef<Record<string, any>>({});+ const enableColumnStats = getApplicationConfig().enableColumnStats;+const fields = useMemo(() => (datasets && dataId ? (datasets[dataId] || {}).fields : undefined),[datasets, dataId]@@ -169,14 +174,15 @@ function DataTableModalFactory(const colMeta = useMemo(() =>fields?.reduce(- (acc, {name, displayName, type, filterProps, format, displayFormat}) => ({+ (acc, {name, displayName, type, filterProps, format, displayFormat, isLoadingStats}) => ({...acc,[name]: {name: displayName || name,type,...(format ? {format} : {}),...(displayFormat ? {displayFormat} : {}),- ...(filterProps?.columnStats ? {columnStats: filterProps.columnStats} : {})+ ...(filterProps?.columnStats ? {columnStats: filterProps.columnStats} : {}),+ ...(typeof isLoadingStats === 'boolean' ? {isLoadingStats} : {})}}),{}@@ -184,51 +190,58 @@ function DataTableModalFactory([fields]);- const cellSizeCache = useMemo(() => {+ const [cellSizeCache, setCellSizeCache] = useState<+ Record<string, {row: number; header: number}>+ >({});++ useEffect(() => {if (!datasets || !dataId || !datasets[dataId]) {- return {};+ setCellSizeCache({});+ return;}- const {fields, dataContainer} = datasets[dataId];- let showCalculate: boolean | null = null;- if (!datasetCellSizeCache.current[dataId]) {- showCalculate = true;- } else if (- datasetCellSizeCache.current[dataId].fields !== fields ||- datasetCellSizeCache.current[dataId].dataContainer !== dataContainer+ const {fields, dataContainer} = datasets[dataId];+ const cached = datasetCellSizeCache.current[dataId];+ if (+ cached &&+ cached.fields === fields &&+ cached.dataContainer === dataContainer &&+ cached.enableColumnStats === enableColumnStats) {- showCalculate = true;- }-- if (!showCalculate) {- return datasetCellSizeCache.current[dataId].cellSizeCache;+ setCellSizeCache(cached.cellSizeCache);+ return;}- const cellSizeCache = fields.reduce(- (acc, field, colIdx) => ({+ const nextCache = fields.reduce((acc, field, colIdx) => {+ const size = renderedSize({+ text: {+ dataContainer,+ column: field.displayName+ },+ colIdx,+ type: field.type,+ fontSize: theme.cellFontSize,+ font: theme.fontFamily+ });+ return {...acc,- [field.name]: renderedSize({- text: {- dataContainer,- column: field.displayName- },- colIdx,- type: field.type,- fontSize: theme.cellFontSize,- font: theme.fontFamily,- minCellSize: MIN_STATS_CELL_SIZE- })- }),- {}- );+ [field.name]: enableColumnStats+ ? {+ row: Math.max(size.row, STATS_WIDTH),+ header: Math.max(size.header, STATS_WIDTH)+ }+ : size+ };+ }, {});datasetCellSizeCache.current[dataId] = {- cellSizeCache,+ cellSizeCache: nextCache,fields,- dataContainer+ dataContainer,+ enableColumnStats};- return cellSizeCache;- }, [dataId, datasets, theme]);+ setCellSizeCache(nextCache);+ }, [dataId, datasets, theme, enableColumnStats]);const handleCopyTableColumn = useCallback((column: string) => {@@ -309,7 +322,8 @@ function DataTableModalFactory(pinTableColumn={handlePinTableColumn}sortTableColumn={handleSortTableColumn}setColumnDisplayFormat={handleSetColumnDisplayFormat}- hasStats={false}+ hasStats={enableColumnStats}+ loadColumnStats={enableColumnStats ? loadColumnStats : undefined}/>) : null}</TableContainer>diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.tsindex 6d62d7a38..2df1f70b1 100644--- a/src/reducers/src/vis-state-updaters.ts+++ b/src/reducers/src/vis-state-updaters.ts@@ -45,7 +45,9 @@ import {fitBounds as fitMapBounds,toggleLayerForMap,applyFilterConfig,- SetLoadingIndicatorPayload+ SetLoadingIndicatorPayload,+ loadColumnStatsSuccess,+ loadColumnStatsError} from '@kepler.gl/actions';// Utils@@ -152,6 +154,7 @@ import {Loader} from '@loaders.gl/loader-utils';import {Datasets,+ FilterProps,assignGpuChannel,copyTableAndUpdate,createNewDataEntry,@@ -188,7 +191,9 @@ import {LayerToFilterTimeInterval,TIME_INTERVALS_ORDERED,mergeFilterDomain,- initCustomPaletteByCustomScale+ initCustomPaletteByCustomScale,+ collectColumnValues,+ getColumnStatistics} from '@kepler.gl/utils';import {createEffect} from '@kepler.gl/effects';import {PayloadAction} from '@reduxjs/toolkit';@@ -4130,6 +4135,123 @@ export function setColumnDisplayFormatUpdater(return newState;}+type LoadColumnStatsTaskPayload = {+ dataset: VisState['datasets'][string];+ field: Field;+};++function getMappedValue(filterProps?: FilterProps | null): unknown[] | undefined {+ if (filterProps && 'mappedValue' in filterProps && Array.isArray(filterProps.mappedValue)) {+ return filterProps.mappedValue;+ }+ return undefined;+}++function yieldForPaint(): Promise<void> {+ return new Promise(resolve => {+ setTimeout(resolve, 0);+ });+}++async function requestDatasetColumnStatistics({dataset, field}: LoadColumnStatsTaskPayload) {+ // Yield so the data table and per-column loading spinners can paint first.+ await yieldForPaint();+ const filterProps = field.filterProps || dataset.getColumnFilterProps(field.name);+ const mappedValue = getMappedValue(filterProps);+ const values = mappedValue ?? collectColumnValues(dataset.dataContainer, field.fieldIdx);+ const result = await getColumnStatistics({+ values,+ fieldType: field.type+ });++ return {result, filterProps};+}++const LOAD_COLUMN_STATS_TASK = Task.fromPromise(+ requestDatasetColumnStatistics,+ 'LOAD_COLUMN_STATS_TASK'+);++/**+ * Start loading column statistics for one or more fields.+ * @memberof visStateUpdaters+ * @public+ */+export function loadColumnStatsUpdater(+ state: VisState,+ {dataId, fieldName}: VisStateActions.LoadColumnStatsUpdaterAction+): VisState {+ const fieldNamesToLoad = toArray(fieldName);+ const dataset = state.datasets[dataId];+ const fieldsToLoad = dataset?.fields.filter(f => fieldNamesToLoad.includes(f.name));+ if (!dataset || !fieldsToLoad.length) {+ return state;+ }++ const newFields = dataset.fields.map(field =>+ fieldNamesToLoad.includes(field.name) ? {...field, isLoadingStats: true} : field+ );+ const newDataset = copyTableAndUpdate(dataset, {fields: newFields});++ const tasks = fieldsToLoad.map(f =>+ LOAD_COLUMN_STATS_TASK({dataset: newDataset, field: f}).bimap(+ ({result, filterProps}) => loadColumnStatsSuccess(newDataset.id, f.name, result, filterProps),+ (error: Error) => loadColumnStatsError(newDataset.id, f.name, error)+ )+ );+ const nextState = pick_('datasets')(merge_({[dataId]: newDataset}))(state);++ return withTask(nextState, tasks);+}++/**+ * Persist loaded column statistics on the field.+ * @memberof visStateUpdaters+ * @public+ */+export function loadColumnStatsSuccessUpdater(+ state: VisState,+ {result, filterProps, dataId, fieldName}: VisStateActions.LoadColumnStatsSuccessUpdaterAction+): VisState {+ const dataset = state.datasets[dataId];+ const field = (dataset?.fields || []).find(f => f.name === fieldName);+ if (!dataset || !field) {+ return state;+ }++ const newFilterProps = {+ ...filterProps,+ columnStats: result+ };+ const newFields = dataset.fields.map(f =>+ f.name === fieldName ? {...f, filterProps: newFilterProps, isLoadingStats: false} : f+ );+ const newDataset = copyTableAndUpdate(dataset, {fields: newFields});+ return pick_('datasets')(merge_({[dataId]: newDataset}))(state);+}++/**+ * Clear loading flag when column statistics fail.+ * @memberof visStateUpdaters+ * @public+ */+export function loadColumnStatsErrorUpdater(+ state: VisState,+ {error, dataId, fieldName}: VisStateActions.LoadColumnStatsErrorUpdaterAction+): VisState {+ Console.error(error);+ const dataset = state.datasets[dataId];+ const field = (dataset?.fields || []).find(f => f.name === fieldName);+ if (!dataset || !field) {+ return state;+ }+ const newFields = dataset.fields.map(f =>+ f.name === fieldName ? {...f, isLoadingStats: false} : f+ );+ const newDataset = copyTableAndUpdate(dataset, {fields: newFields});+ return pick_('datasets')(merge_({[dataId]: newDataset}))(state);+}+/*** Update editor*/diff --git a/src/reducers/src/vis-state.ts b/src/reducers/src/vis-state.tsindex 5e99082fd..88816d2f6 100644--- a/src/reducers/src/vis-state.ts+++ b/src/reducers/src/vis-state.ts@@ -158,6 +158,12 @@ const actionHandler = {[ActionTypes.SET_COLUMN_DISPLAY_FORMAT]: visStateUpdaters.setColumnDisplayFormatUpdater,+ [ActionTypes.LOAD_COLUMN_STATS]: visStateUpdaters.loadColumnStatsUpdater,++ [ActionTypes.LOAD_COLUMN_STATS_SUCCESS]: visStateUpdaters.loadColumnStatsSuccessUpdater,++ [ActionTypes.LOAD_COLUMN_STATS_ERROR]: visStateUpdaters.loadColumnStatsErrorUpdater,+[ActionTypes.NEXT_FILE_BATCH]: visStateUpdaters.nextFileBatchUpdater,[ActionTypes.PROCESS_FILE_CONTENT]: visStateUpdaters.processFileContentUpdater,diff --git a/src/styles/src/base.ts b/src/styles/src/base.tsindex 884102b27..2cb275a01 100644--- a/src/styles/src/base.ts+++ b/src/styles/src/base.ts@@ -386,7 +386,7 @@ export const notificationPanelItemWidth = notificationPanelWidth - 60;export const notificationPanelItemHeight = 60;// Data Table-const headerRowHeight = 70;+const headerRowHeight = 40;const headerStatsControlHeight = 32;const headerRowWStatsHeight = 364;const rowHeight = 32;@@ -1093,36 +1093,26 @@ const scrollBar = css`export const modalScrollBar = css`&::-webkit-scrollbar {width: 14px;- height: 16px;+ height: 14px;}&::-webkit-scrollbar-track {background: white;}- &::-webkit-scrollbar-track:horizontal {- background: ${props => props.theme.textColorHl};- }+&::-webkit-scrollbar-thumb {background: ${props => props.theme.scrollbarThumbColorLT};border: 4px solid white;+ border-radius: 7px;}&::-webkit-scrollbar-corner {- background: ${props => props.theme.textColorHl};+ background: white;}&::-webkit-scrollbar-thumb:hover {background: ${props => props.theme.scrollbarThumbColorHoverLT};}-- &::-webkit-scrollbar-thumb:vertical {- border-radius: 7px;- }-- &::-webkit-scrollbar-thumb:horizontal {- border-radius: 9px;- border: 4px solid ${props => props.theme.textColorHl};- }`;export const theme = {diff --git a/src/types/components.d.ts b/src/types/components.d.tsindex 734541270..4745c91b3 100644--- a/src/types/components.d.ts+++ b/src/types/components.d.ts@@ -26,6 +26,7 @@ export type ColMetaProps = {format?: string;columnStats?: any;displayFormat?: string;+ isLoadingStats?: boolean;};export type ColMeta = {diff --git a/src/utils/src/application-config.ts b/src/utils/src/application-config.tsindex 5bd5d0eba..047383098 100644--- a/src/utils/src/application-config.ts+++ b/src/utils/src/application-config.ts@@ -177,6 +177,9 @@ export type KeplerApplicationConfig = {/** Whether to show a map control to toggle between light and dark UI themes. Disabled by default. */enableThemeToggle?: boolean;+ /** Whether to show column statistics in the data table modal. Enabled by default. */+ enableColumnStats?: boolean;+/*** Custom SVG icons to be made available in the icon layer.* These icons will be merged with the default icons fetched from CDN.@@ -302,11 +305,13 @@ const DEFAULT_APPLICATION_CONFIG: Required<KeplerApplicationConfig> = {enableThemeToggle: false,+ enableColumnStats: true,+customIcons: [],customIconUrl: '',- reduxLogLevel: 1,+ reduxLogLevel: 1};const applicationConfig: Required<KeplerApplicationConfig> = DEFAULT_APPLICATION_CONFIG;diff --git a/src/utils/src/column-statistics.ts b/src/utils/src/column-statistics.tsnew file mode 100644index 000000000..e6a8995a5--- /dev/null+++ b/src/utils/src/column-statistics.ts@@ -0,0 +1,162 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import {deviation, mean, quantileSorted} from 'd3-array';++import {ALL_FIELD_TYPES} from '@kepler.gl/constants';+import {Bin} from '@kepler.gl/types';++import {isNumber} from './data-utils';+import {histogramFromValues} from './plot';+import quickInsertionSort from './quick-insertion-sort';+import {DataContainerInterface} from './data-container-interface';++export const NUMERIC_BIN_COUNT = 20;++export enum ColumnStatsType {+ numeric = 'numeric',+ categorical = 'categorical',+ boolean = 'boolean',+ time = 'time',+ geo = 'geo',+ other = 'other'+}++export type Quantile = {label: string; value: number | null};++export type NumericColumnStats = {+ type: ColumnStatsType.numeric;+ bins: Bin[];+ quantiles: Quantile[];+ mean: number | null;+ std: number | null;+ percentNulls: number | null;+};++export type CategoricalColumnStats = {+ type: ColumnStatsType.categorical;+ uniqueValues: number;+ percentNulls: number | null;+};++export type TimeColumnStats = {+ type: ColumnStatsType.time;+ bins: Bin[];+ min: number | null;+ max: number | null;+ percentNulls: number | null;+};++export type BooleanColumnStats = {+ type: ColumnStatsType.boolean;+};++export type GeometryColumnStats = {+ type: ColumnStatsType.geo;+};++export type UnknownColumnStats = {+ type: ColumnStatsType.other;+};++export type ColumnStats =+ | NumericColumnStats+ | BooleanColumnStats+ | CategoricalColumnStats+ | TimeColumnStats+ | GeometryColumnStats+ | UnknownColumnStats;++const QUANTILE_STOPS = [0, 0.25, 0.5, 0.75, 1];++function percentNulls(total: number, valid: number): number | null {+ return total > 0 ? (total - valid) / total : null;+}++function getNumericStats(values: readonly unknown[]): NumericColumnStats {+ const filtered = values.filter(isNumber);+ quickInsertionSort(filtered);++ const quantiles = QUANTILE_STOPS.map(p => ({+ label: p === 0 ? 'Min' : p === 1 ? 'Max' : `${p * 100}%`,+ value: filtered.length ? quantileSorted(filtered, p) ?? null : null+ }));++ return {+ type: ColumnStatsType.numeric,+ bins: histogramFromValues(filtered, NUMERIC_BIN_COUNT),+ quantiles,+ percentNulls: percentNulls(values.length, filtered.length),+ mean: filtered.length ? mean(filtered) ?? null : null,+ std: filtered.length ? deviation(filtered) ?? null : null+ };+}++function isPresentCategoricalValue(v: unknown): boolean {+ return v !== null && v !== undefined && v !== '';+}++function getCategoricalStats(values: readonly unknown[]): CategoricalColumnStats {+ const filtered = values.filter(isPresentCategoricalValue);+ return {+ type: ColumnStatsType.categorical,+ uniqueValues: new Set(filtered).size,+ percentNulls: percentNulls(values.length, filtered.length)+ };+}++function getTimeStats(values: readonly unknown[]): TimeColumnStats {+ const filtered = values.filter(isNumber);+ const bins = histogramFromValues(filtered, NUMERIC_BIN_COUNT);++ return {+ type: ColumnStatsType.time,+ bins,+ min: filtered.length ? filtered.reduce((a, b) => (a < b ? a : b)) : null,+ max: filtered.length ? filtered.reduce((a, b) => (a > b ? a : b)) : null,+ percentNulls: percentNulls(values.length, filtered.length)+ };+}++export function collectColumnValues(+ dataContainer: DataContainerInterface,+ fieldIdx: number+): unknown[] {+ const n = dataContainer.numRows();+ const values = new Array(n);+ for (let i = 0; i < n; i++) {+ values[i] = dataContainer.valueAt(i, fieldIdx);+ }+ return values;+}++export function computeColumnStats(values: readonly unknown[], fieldType: string): ColumnStats {+ switch (fieldType) {+ case ALL_FIELD_TYPES.real:+ case ALL_FIELD_TYPES.integer:+ return getNumericStats(values);+ case ALL_FIELD_TYPES.string:+ case ALL_FIELD_TYPES.date:+ case ALL_FIELD_TYPES.boolean:+ return getCategoricalStats(values);+ case ALL_FIELD_TYPES.timestamp:+ return getTimeStats(values);+ default:+ return {type: ColumnStatsType.other};+ }+}++/**+ * Compute column statistics asynchronously so the UI can paint a loading state first.+ */+export function getColumnStatistics({+ values,+ fieldType+}: {+ values: readonly unknown[];+ fieldType: string;+}): Promise<ColumnStats> {+ return new Promise(resolve => {+ resolve(computeColumnStats(values, fieldType));+ });+}diff --git a/src/utils/src/index.ts b/src/utils/src/index.tsindex df90ae725..b38a85654 100644--- a/src/utils/src/index.ts+++ b/src/utils/src/index.ts@@ -33,6 +33,20 @@ export {createNotification, exportImageError, successNotification} from './notifexport {setStyleSheetBaseHref} from './dom-utils';export {default as domtoimage} from './dom-to-image';export {getFrequency, getMode, aggregate} from './aggregation';+export {+ ColumnStatsType,+ NUMERIC_BIN_COUNT,+ collectColumnValues,+ computeColumnStats,+ getColumnStatistics+} from './column-statistics';+export type {+ ColumnStats,+ NumericColumnStats,+ CategoricalColumnStats,+ TimeColumnStats,+ Quantile+} from './column-statistics';export {adjustValueToAnimationWindow,getBinThresholds,diff --git a/src/utils/src/plot.ts b/src/utils/src/plot.tsindex d5475c65d..52f549ea0 100644--- a/src/utils/src/plot.ts+++ b/src/utils/src/plot.ts@@ -113,7 +113,7 @@ export function histogramFromValues(values: (Millisecond | null | number)[],numBins: number,valueAccessor?: (d: number) => number-) {+): Bin[] {const getBins = d3Histogram().thresholds(numBins);if (valueAccessor) {@@ -131,7 +131,7 @@ export function histogramFromValues(.filter(b => {const {x0, x1} = b;return isNumber(x0) && isNumber(x1);- });+ }) as Bin[];}export function histogramFromOrdinal(diff --git a/test/browser/components/modals/data-table-modal-test.js b/test/browser/components/modals/data-table-modal-test.jsindex 65bfd645f..0e29af0a7 100644--- a/test/browser/components/modals/data-table-modal-test.js+++ b/test/browser/components/modals/data-table-modal-test.js@@ -32,7 +32,7 @@ import {geoStyleFields, geoStyleRows} from 'test/fixtures/geojson';import {StateWFiles, testCsvDataId, testGeoJsonDataId} from 'test/helpers/mock-state';import {STYLED_COMPONENTS_DUPLICATED_ENTRIES} from '../../../helpers/utils';-import {createDataContainer, getFieldFormatLabels} from '@kepler.gl/utils';+import {createDataContainer, getFieldFormatLabels, initApplicationConfig} from '@kepler.gl/utils';const {VertThreeDots} = Icons;const DataTableModal = appInjector.get(DataTableModalFactory);@@ -246,6 +246,7 @@ test('Components -> DataTableModal.render: csv 1', t => {t.deepEqual(props.columns, expectedColumns, 'DataTable should have the correct props.columns');t.deepEqual(props.colMeta, expectedColMeta, 'DataTable should have the correct props.colMeta');+ t.equal(props.hasStats, true, 'DataTable should enable column stats by default');const datasetId = Object.keys(StateWFiles.visState.datasets)[0];@@ -264,6 +265,20 @@ test('Components -> DataTableModal.render: csv 1', t => {t.end();});+test('Components -> DataTableModal.render: disableColumnStats', t => {+ initApplicationConfig({enableColumnStats: false});+ try {+ const wrapper = mountWithTheme(+ <DataTableModal datasets={StateWFiles.visState.datasets} dataId={testCsvDataId} />+ );+ const props = wrapper.find(DataTable).at(0).props();+ t.equal(props.hasStats, false, 'DataTable should disable column stats when config flag is off');+ } finally {+ initApplicationConfig({enableColumnStats: true});+ }+ t.end();+});+test('Components -> DataTableModal -> click tab', t => {const showDatasetTable = sinon.spy();@@ -276,7 +291,11 @@ test('Components -> DataTableModal -> click tab', t => {);t.equal(wrapper.find(DatasetModalTab).length, 2, 'should render 2 DatasetModalTab');- t.equal(wrapper.find(DatasetModalTab).at(0).prop('$active'), true, 'prop 0 active should be true');+ t.equal(+ wrapper.find(DatasetModalTab).at(0).prop('$active'),+ true,+ 'prop 0 active should be true'+ );t.equal(wrapper.find(DatasetModalTab).at(0).find('.dataset-name').text(),'hello.csv',@@ -567,47 +586,50 @@ test('Components -> DatableModal -> sort/pin/copy and display format should be ctest('Components -> cellSize -> renderedSize', t => {prepareMockCanvas();+ initApplicationConfig({enableColumnStats: true});+ try {+ const fields = [+ {name: testColumns[0], type: 'geojson'},+ {name: testColumns[1], type: 'real'},+ {name: testColumns[2], type: 'string'}+ ];+ const dataContainer = createDataContainer(texts, {fields});++ const wrapper = mountWithTheme(+ <DataTableModal+ datasets={{+ smoothie: {+ id: 'smoothie',+ dataContainer,+ fields: [+ {name: testColumns[0], type: 'geojson', displayName: testColumns[0]},+ {name: testColumns[1], type: 'real', displayName: testColumns[1]},+ {name: testColumns[2], type: 'string', displayName: testColumns[2]}+ ],+ color: [113, 113, 113]+ }+ }}+ dataId="smoothie"+ />+ );- const fields = [- {name: testColumns[0], type: 'geojson'},- {name: testColumns[1], type: 'real'},- {name: testColumns[2], type: 'string'}- ];- const dataContainer = createDataContainer(texts, {fields});-- const wrapper = mountWithTheme(- <DataTableModal- datasets={{- smoothie: {- id: 'smoothie',- dataContainer,- fields: [- {name: testColumns[0], type: 'geojson', displayName: testColumns[0]},- {name: testColumns[1], type: 'real', displayName: testColumns[1]},- {name: testColumns[2], type: 'string', displayName: testColumns[2]}- ],- color: [113, 113, 113]- }- }}- dataId="smoothie"- />- );-- const props = wrapper.find(DataTable).at(0).props();-- const expected = {- _geojson: {row: 500, header: 186},- 'income level of people over 65': {row: 162, header: 223},- engagement: {row: 162, header: 186}- };+ const props = wrapper.find(DataTable).at(0).props();- t.deepEqual(- props.cellSizeCache,- expected,- 'DataTable should have the correct props.cellSizeCache'- );+ const expected = {+ _geojson: {row: 180, header: 180},+ 'income level of people over 65': {row: 180, header: 223},+ engagement: {row: 180, header: 180}+ };- restoreMockCanvas();+ t.deepEqual(+ props.cellSizeCache,+ expected,+ 'DataTable should have the correct props.cellSizeCache'+ );+ } finally {+ restoreMockCanvas();+ initApplicationConfig({enableColumnStats: true});+ }t.end();});diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex 8c059883e..dc3375a01 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -6230,6 +6230,62 @@ test('#visStateReducer -> SORT_TABLE_COLUMN', t => {t.end();});+test('#visStateReducer -> LOAD_COLUMN_STATS', t => {+ drainTasksForTesting();+ const initialState = CloneDeep(StateWFiles.visState);++ const noOpState = reducer(+ initialState,+ VisStateActions.loadColumnStats('missing', 'gps_data.lat')+ );+ t.equal(noOpState, initialState, 'state should not change when dataset is missing');++ const nextState = reducer(+ initialState,+ VisStateActions.loadColumnStats(testCsvDataId, 'gps_data.lat')+ );+ const loadingField = nextState.datasets[testCsvDataId].fields.find(f => f.name === 'gps_data.lat');+ t.equal(loadingField.isLoadingStats, true, 'should set isLoadingStats while loading');++ const tasks = drainTasksForTesting();+ t.equal(tasks.length, 1, 'should create a column stats task');++ const result = {+ type: 'numeric',+ mean: 30,+ std: 1,+ percentNulls: 0,+ bins: [],+ quantiles: [+ {label: 'Min', value: 29},+ {label: 'Max', value: 31}+ ]+ };+ const successState = reducer(+ nextState,+ VisStateActions.loadColumnStatsSuccess(testCsvDataId, 'gps_data.lat', result, {})+ );+ const successField = successState.datasets[testCsvDataId].fields.find(+ f => f.name === 'gps_data.lat'+ );+ t.equal(successField.isLoadingStats, false, 'should clear isLoadingStats on success');+ t.deepEqual(+ successField.filterProps.columnStats,+ result,+ 'should store columnStats on filterProps'+ );++ const errorState = reducer(+ nextState,+ VisStateActions.loadColumnStatsError(testCsvDataId, 'gps_data.lat', new Error('stats failed'))+ );+ const errorField = errorState.datasets[testCsvDataId].fields.find(f => f.name === 'gps_data.lat');+ t.equal(errorField.isLoadingStats, false, 'should clear isLoadingStats on error');++ drainTasksForTesting();+ t.end();+});+test('#visStateReducer -> PIN_TABLE_COLUMN', t => {const initialState = CloneDeep(StateWFiles.visState);diff --git a/test/node/utils/column-statistics-test.js b/test/node/utils/column-statistics-test.jsnew file mode 100644index 000000000..9d3528cfe--- /dev/null+++ b/test/node/utils/column-statistics-test.js@@ -0,0 +1,72 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import test from 'tape';++import {ALL_FIELD_TYPES} from '@kepler.gl/constants';+import {ColumnStatsType, computeColumnStats, getColumnStatistics} from '@kepler.gl/utils';++test('columnStatistics -> numeric', t => {+ const stats = computeColumnStats([1, 2, 3, 4, null], ALL_FIELD_TYPES.integer);++ t.equal(stats.type, ColumnStatsType.numeric, 'should return numeric stats');+ if (stats.type === ColumnStatsType.numeric) {+ t.equal(stats.mean, 2.5, 'should compute mean from valid numbers');+ t.equal(stats.percentNulls, 0.2, 'should compute percent nulls');+ t.equal(stats.quantiles[0].label, 'Min', 'should include min quantile');+ t.equal(stats.quantiles[0].value, 1, 'min should be 1');+ t.equal(stats.quantiles[stats.quantiles.length - 1].value, 4, 'max should be 4');+ t.ok(stats.bins.length > 0, 'should compute histogram bins');+ }+ t.end();+});++test('columnStatistics -> categorical', t => {+ const stats = computeColumnStats(['a', 'b', 'a', '', null], ALL_FIELD_TYPES.string);++ t.equal(stats.type, ColumnStatsType.categorical, 'should return categorical stats');+ if (stats.type === ColumnStatsType.categorical) {+ t.equal(stats.uniqueValues, 2, 'should count unique non-empty strings');+ t.equal(stats.percentNulls, 0.4, 'should treat empty string and null as nulls');+ }+ t.end();+});++test('columnStatistics -> boolean as categorical', t => {+ const stats = computeColumnStats([true, false, true, null], ALL_FIELD_TYPES.boolean);++ t.equal(stats.type, ColumnStatsType.categorical, 'should treat boolean as categorical');+ if (stats.type === ColumnStatsType.categorical) {+ t.equal(stats.uniqueValues, 2, 'should count true and false as unique values');+ t.equal(stats.percentNulls, 0.25, 'should compute percent nulls');+ }+ t.end();+});++test('columnStatistics -> timestamp', t => {+ const stats = computeColumnStats([1000, 3000, 2000], ALL_FIELD_TYPES.timestamp);++ t.equal(stats.type, ColumnStatsType.time, 'should return time stats');+ if (stats.type === ColumnStatsType.time) {+ t.equal(stats.min, 1000, 'should compute min timestamp');+ t.equal(stats.max, 3000, 'should compute max timestamp');+ t.equal(stats.percentNulls, 0, 'should have no nulls');+ t.ok(stats.bins.length > 0, 'should compute histogram bins');+ }+ t.end();+});++test('columnStatistics -> other', t => {+ const stats = computeColumnStats(['{"type":"Point"}'], ALL_FIELD_TYPES.geojson);+ t.equal(stats.type, ColumnStatsType.other, 'should return other for geojson');+ t.end();+});++test('columnStatistics -> getColumnStatistics is async', async t => {+ const stats = await getColumnStatistics({+ values: [1, 2, 3],+ fieldType: ALL_FIELD_TYPES.real+ });+ t.equal(stats.type, ColumnStatsType.numeric, 'should resolve numeric stats');+ t.end();+});diff --git a/test/node/utils/index.js b/test/node/utils/index.jsindex 0958ec375..724d623c3 100644--- a/test/node/utils/index.js+++ b/test/node/utils/index.js@@ -2,6 +2,7 @@// Copyright contributors to the kepler.gl projectimport './data-utils-test';+import './column-statistics-test';import './data-processor-test';import './kepler-table-test';import './kepler-table-utils-test';