A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
improve performant table charts, remove feat flag (#6892)
Shahmir Varqha15b793ae9913
diff --git a/frontend/src/components/data-table/__tests__/__snapshots__/chart-spec-model.test.ts.snap b/frontend/src/components/data-table/__tests__/__snapshots__/chart-spec-model.test.ts.snapindex 6d2f56d945..f397e4f653 100644--- a/frontend/src/components/data-table/__tests__/__snapshots__/chart-spec-model.test.ts.snap+++ b/frontend/src/components/data-table/__tests__/__snapshots__/chart-spec-model.test.ts.snap@@ -123,7 +123,7 @@ exports[`ColumnChartSpecModel > file URL handling > should handle arrow data 1`]}`;-exports[`ColumnChartSpecModel > should expect bin values to be used for number and integer columns when feat flag is true 1`] = `+exports[`ColumnChartSpecModel > should expect bin values to be used for number and integer columns 1`] = `{"background": "transparent","config": {@@ -153,13 +153,14 @@ exports[`ColumnChartSpecModel > should expect bin values to be used for number a"layer": [{"encoding": {- "strokeWidth": {- "condition": {- "empty": false,- "param": "hover",- "value": 0.5,- },- "value": 0,+ "opacity": {+ "condition": [+ {+ "param": "hover",+ "value": 1,+ },+ ],+ "value": 0.6,},"x": {"bin": {@@ -181,20 +182,8 @@ exports[`ColumnChartSpecModel > should expect bin values to be used for number a},"mark": {"color": "#027864",- "stroke": "#027864",- "strokeWidth": 0,"type": "bar",},- "params": [- {- "name": "hover",- "select": {- "clear": "mouseout",- "on": "mouseover",- "type": "point",- },- },- ],},{"encoding": {@@ -242,6 +231,17 @@ exports[`ColumnChartSpecModel > should expect bin values to be used for number a"opacity": 0,"type": "bar",},+ "params": [+ {+ "name": "hover",+ "select": {+ "clear": "mouseout",+ "nearest": true,+ "on": "mouseover",+ "type": "point",+ },+ },+ ],"transform": [{"as": "bin_range",@@ -254,7 +254,7 @@ exports[`ColumnChartSpecModel > should expect bin values to be used for number a}`;-exports[`ColumnChartSpecModel > should handle boolean stats when feat flag is true 1`] = `+exports[`ColumnChartSpecModel > should handle boolean stats 1`] = `{"background": "transparent","config": {@@ -378,16 +378,13 @@ exports[`ColumnChartSpecModel > should handle boolean stats when feat flag is tr}`;-exports[`ColumnChartSpecModel > should handle datetime bin values when feat flag is true 1`] = `+exports[`ColumnChartSpecModel > should handle datetime bin values 1`] = `{"background": "transparent","config": {"axis": {"domain": false,},- "concat": {- "spacing": 0,- },"view": {"stroke": "transparent",},@@ -412,177 +409,98 @@ exports[`ColumnChartSpecModel > should handle datetime bin values when feat flag},],},- "hconcat": [+ "height": 100,+ "layer": [{- "height": 30,- "layer": [- {- "encoding": {- "x": {- "axis": null,- "field": "bin_start",- "type": "nominal",- },- "y": {- "axis": null,- "field": "count",- "type": "quantitative",- },- },- "mark": {- "color": "#cc4e00",- "type": "bar",+ "encoding": {+ "color": {+ "condition": {+ "test": "datum['bin_start'] === null && datum['bin_end'] === null",+ "value": "#cc4e00",},+ "value": "#027864",},- {- "encoding": {- "tooltip": [- {- "field": "count",- "format": ",d",- "title": "nulls",- "type": "quantitative",- },- ],- "x": {- "axis": null,- "field": "bin_start",- "type": "nominal",- },- "y": {- "aggregate": "max",- "axis": null,- "type": "quantitative",+ "opacity": {+ "condition": [+ {+ "param": "hover",+ "value": 1,},- },- "mark": {- "opacity": 0,- "type": "bar",- },+ ],+ "value": 0.6,},- ],- "transform": [+ "x": {+ "axis": null,+ "field": "bin_start",+ "type": "ordinal",+ },+ "y": {+ "axis": null,+ "field": "count",+ "type": "quantitative",+ },+ },+ "mark": {+ "color": "#027864",+ "type": "bar",+ },+ "params": [{- "filter": "datum['bin_start'] === null && datum['bin_end'] === null",+ "name": "hover",+ "select": {+ "clear": "mouseout",+ "on": "mouseover",+ "type": "point",+ },},],- "width": 5,},{- "height": 30,- "layer": [- {- "encoding": {- "strokeWidth": {- "condition": {- "empty": false,- "param": "hover",- "value": 0.5,- },- "value": 0,- },- "x": {- "axis": null,- "bin": {- "binned": true,- "step": 86400000,- },- "field": "bin_start",- "type": "temporal",- },- "x2": {- "axis": null,- "field": "bin_end",- "type": "temporal",- },- "y": {- "axis": null,- "field": "count",- "type": "quantitative",- },- },- "mark": {- "color": "#027864",- "stroke": "#027864",- "strokeWidth": 0,- "type": "bar",+ "encoding": {+ "tooltip": [+ {+ "field": "bin_start",+ "timeUnit": "yearmonthdate",+ "title": "datetime (start)",+ "type": "temporal",},- "params": [- {- "name": "hover",- "select": {- "clear": "mouseout",- "on": "mouseover",- "type": "point",- },- },- ],- },- {- "encoding": {- "tooltip": [- {- "field": "bin_start",- "timeUnit": "yearmonthdate",- "title": "datetime (start)",- "type": "temporal",- },- {- "field": "bin_end",- "timeUnit": "yearmonthdate",- "title": "datetime (end)",- "type": "temporal",- },- {- "field": "count",- "format": ",d",- "title": "Count",- "type": "quantitative",- },- ],- "x": {- "axis": null,- "bin": {- "binned": true,- "step": 86400000,- },- "field": "bin_start",- "type": "temporal",- },- "x2": {- "axis": null,- "bin": {- "binned": true,- "step": 86400000,- },- "field": "bin_end",- "type": "temporal",- },- "y": {- "aggregate": "max",- "axis": null,- "type": "quantitative",- },+ {+ "field": "bin_end",+ "timeUnit": "yearmonthdate",+ "title": "datetime (end)",+ "type": "temporal",},- "mark": {- "opacity": 0,- "type": "bar",+ {+ "field": "count",+ "format": ",d",+ "title": "Count",+ "type": "quantitative",},+ ],+ "x": {+ "axis": null,+ "field": "bin_start",+ "type": "ordinal",},- ],- "width": 70,+ "y": {+ "aggregate": "max",+ "axis": null,+ "type": "quantitative",+ },+ },+ "mark": {+ "opacity": 0,+ "type": "bar",+ "width": {+ "band": 1.2,+ },+ },},],- "height": 100,- "resolve": {- "scale": {- "y": "shared",- },- },}`;-exports[`ColumnChartSpecModel > should handle string value counts when feat flag is true 1`] = `+exports[`ColumnChartSpecModel > should handle string value counts 1`] = `{"background": "transparent","config": {@@ -619,18 +537,20 @@ exports[`ColumnChartSpecModel > should handle string value counts when feat flag{"encoding": {"color": {+ "condition": {+ "test": "datum.value == "None" || datum.value == "null"",+ "value": "#cc4e00",+ },+ "value": "#027864",+ },+ "opacity": {"condition": [{"param": "hover_bar",- "value": "#027864",- },- {- "test": "datum.value == "None" || datum.value == "null"",- "value": "#cc4e00",+ "value": 1,},],- "legend": null,- "value": "#4cbba5",+ "value": 0.6,},"tooltip": [{@@ -1077,3 +997,372 @@ exports[`ColumnChartSpecModel > snapshot > url data 1`] = `],}`;++exports[`ColumnChartSpecModel > snapshot with legacy data spec > array 1`] = `+{+ "background": "transparent",+ "config": {+ "axis": {+ "domain": false,+ },+ "view": {+ "stroke": "transparent",+ },+ },+ "data": {+ "values": [+ "a",+ "b",+ "c",+ ],+ },+ "height": 100,+ "layer": [+ {+ "encoding": {+ "x": {+ "bin": true,+ "field": "a",+ "type": "quantitative",+ },+ "y": {+ "aggregate": "count",+ "axis": null,+ "type": "quantitative",+ },+ },+ "mark": {+ "color": "#027864",+ "type": "bar",+ },+ },+ {+ "encoding": {+ "tooltip": [+ {+ "bin": true,+ "field": "a",+ "format": ".2f",+ "title": "a",+ "type": "quantitative",+ },+ {+ "aggregate": "count",+ "format": ",d",+ "title": "Count",+ "type": "quantitative",+ },+ ],+ "x": {+ "axis": {+ "labelExpr": "(datum.value >= 10000 || datum.value <= -10000) ? format(datum.value, '.2e') : format(datum.value, '.2~f')",+ "labelFontSize": 8.5,+ "labelOpacity": 0.5,+ "title": null,+ },+ "bin": true,+ "field": "a",+ "type": "quantitative",+ },+ "y": {+ "aggregate": "max",+ "axis": null,+ "type": "quantitative",+ },+ },+ "mark": {+ "opacity": 0,+ "type": "bar",+ },+ },+ ],+}+`;++exports[`ColumnChartSpecModel > snapshot with legacy data spec > csv data 1`] = `+{+ "background": "transparent",+ "config": {+ "axis": {+ "domain": false,+ },+ "view": {+ "stroke": "transparent",+ },+ },+ "data": {+ "values": [+ {+ "a": 1,+ "b": 2,+ "c": 3,+ },+ {+ "a": 4,+ "b": 5,+ "c": 6,+ },+ ],+ },+ "height": 100,+ "layer": [+ {+ "encoding": {+ "x": {+ "bin": true,+ "field": "a",+ "type": "quantitative",+ },+ "y": {+ "aggregate": "count",+ "axis": null,+ "type": "quantitative",+ },+ },+ "mark": {+ "color": "#027864",+ "type": "bar",+ },+ },+ {+ "encoding": {+ "tooltip": [+ {+ "bin": true,+ "field": "a",+ "format": ".2f",+ "title": "a",+ "type": "quantitative",+ },+ {+ "aggregate": "count",+ "format": ",d",+ "title": "Count",+ "type": "quantitative",+ },+ ],+ "x": {+ "axis": {+ "labelExpr": "(datum.value >= 10000 || datum.value <= -10000) ? format(datum.value, '.2e') : format(datum.value, '.2~f')",+ "labelFontSize": 8.5,+ "labelOpacity": 0.5,+ "title": null,+ },+ "bin": true,+ "field": "a",+ "type": "quantitative",+ },+ "y": {+ "aggregate": "max",+ "axis": null,+ "type": "quantitative",+ },+ },+ "mark": {+ "opacity": 0,+ "type": "bar",+ },+ },+ ],+}+`;++exports[`ColumnChartSpecModel > snapshot with legacy data spec > csv string 1`] = `+{+ "background": "transparent",+ "config": {+ "axis": {+ "domain": false,+ },+ "view": {+ "stroke": "transparent",+ },+ },+ "data": {+ "values": [+ {+ "a": 1,+ "b": 2,+ "c": 3,+ },+ {+ "a": 4,+ "b": 5,+ "c": 6,+ },+ ],+ },+ "height": 100,+ "layer": [+ {+ "encoding": {+ "x": {+ "bin": true,+ "field": "a",+ "type": "quantitative",+ },+ "y": {+ "aggregate": "count",+ "axis": null,+ "type": "quantitative",+ },+ },+ "mark": {+ "color": "#027864",+ "type": "bar",+ },+ },+ {+ "encoding": {+ "tooltip": [+ {+ "bin": true,+ "field": "a",+ "format": ".2f",+ "title": "a",+ "type": "quantitative",+ },+ {+ "aggregate": "count",+ "format": ",d",+ "title": "Count",+ "type": "quantitative",+ },+ ],+ "x": {+ "axis": {+ "labelExpr": "(datum.value >= 10000 || datum.value <= -10000) ? format(datum.value, '.2e') : format(datum.value, '.2~f')",+ "labelFontSize": 8.5,+ "labelOpacity": 0.5,+ "title": null,+ },+ "bin": true,+ "field": "a",+ "type": "quantitative",+ },+ "y": {+ "aggregate": "max",+ "axis": null,+ "type": "quantitative",+ },+ },+ "mark": {+ "opacity": 0,+ "type": "bar",+ },+ },+ ],+}+`;++exports[`ColumnChartSpecModel > snapshot with legacy data spec > url data 1`] = `+{+ "background": "transparent",+ "config": {+ "axis": {+ "domain": false,+ },+ "view": {+ "stroke": "transparent",+ },+ },+ "data": {+ "values": [],+ },+ "height": 100,+ "layer": [+ {+ "encoding": {+ "color": {+ "condition": {+ "test": "datum["bin_maxbins_10_date_range"] === "null"",+ "value": "#cc4e00",+ },+ "value": "#027864",+ },+ "x": {+ "axis": null,+ "bin": true,+ "field": "date",+ "scale": {+ "align": 0,+ "paddingInner": 0,+ "paddingOuter": {+ "expr": "length(data('data_0')) == 2 ? 1 : length(data('data_0')) == 3 ? 0.5 : length(data('data_0')) == 4 ? 0 : 0",+ },+ },+ "type": "temporal",+ },+ "y": {+ "aggregate": "count",+ "axis": null,+ "type": "quantitative",+ },+ },+ "mark": {+ "color": "#027864",+ "type": "bar",+ },+ },+ {+ "encoding": {+ "color": {+ "condition": {+ "test": "datum["bin_maxbins_10_date_range"] === "null"",+ "value": "#cc4e00",+ },+ "value": "#027864",+ },+ "tooltip": [+ {+ "bin": {+ "binned": true,+ },+ "field": "bin_maxbins_10_date",+ "format": "%Y-%m-%d",+ "title": "date (start)",+ "type": "temporal",+ },+ {+ "bin": {+ "binned": true,+ },+ "field": "bin_maxbins_10_date_end",+ "format": "%Y-%m-%d",+ "title": "date (end)",+ "type": "temporal",+ },+ {+ "aggregate": "count",+ "format": ",d",+ "title": "Count",+ "type": "quantitative",+ },+ ],+ "x": {+ "axis": null,+ "bin": true,+ "field": "date",+ "scale": {+ "align": 0,+ "paddingInner": 0,+ "paddingOuter": {+ "expr": "length(data('data_0')) == 2 ? 1 : length(data('data_0')) == 3 ? 0.5 : length(data('data_0')) == 4 ? 0 : 0",+ },+ },+ "type": "temporal",+ },+ "y": {+ "aggregate": "max",+ "axis": null,+ "type": "quantitative",+ },+ },+ "mark": {+ "opacity": 0,+ "type": "bar",+ },+ },+ ],+}+`;diff --git a/frontend/src/components/data-table/__tests__/chart-spec-model.test.ts b/frontend/src/components/data-table/__tests__/chart-spec-model.test.tsindex f7c920650f..1b581980c8 100644--- a/frontend/src/components/data-table/__tests__/chart-spec-model.test.ts+++ b/frontend/src/components/data-table/__tests__/chart-spec-model.test.ts@@ -61,7 +61,7 @@ describe("ColumnChartSpecModel", () => {it("should create an instance", () => {const model = new ColumnChartSpecModel(- mockData,+ [],mockFieldTypes,mockStats,mockBinValues,@@ -78,7 +78,7 @@ describe("ColumnChartSpecModel", () => {it("should return header summary with spec when includeCharts is true", () => {const model = new ColumnChartSpecModel(- mockData,+ [],mockFieldTypes,mockStats,mockBinValues,@@ -93,7 +93,7 @@ describe("ColumnChartSpecModel", () => {it("should return header summary without spec when includeCharts is false", () => {const model = new ColumnChartSpecModel(- mockData,+ [],mockFieldTypes,mockStats,mockBinValues,@@ -106,17 +106,17 @@ describe("ColumnChartSpecModel", () => {expect(numberSummary.spec).toBeUndefined();});- it("should return null spec for string and unknown types", () => {+ it("should return null spec for unknown types", () => {const model = new ColumnChartSpecModel(- mockData,+ [],mockFieldTypes,mockStats,mockBinValues,mockValueCounts,{ includeCharts: true },);- const stringSummary = model.getHeaderSummary("string");- expect(stringSummary.spec).toBeNull();+ const unknownSummary = model.getHeaderSummary("unknown");+ expect(unknownSummary.spec).toBeNull();});it("should handle special characters in column names", () => {@@ -126,11 +126,18 @@ describe("ColumnChartSpecModel", () => {const specialStats: Record<ColumnName, Partial<ColumnHeaderStats>> = {"column.with[special:chars]": { min: "2023-01-01", max: "2023-12-31" },};+ const specialBinValues: Record<ColumnName, BinValues> = {+ ...mockBinValues,+ "column.with[special:chars]": [+ { bin_start: "2023-01-01", bin_end: "2023-06-01", count: 10 },+ { bin_start: "2023-06-01", bin_end: "2023-12-31", count: 20 },+ ],+ };const model = new ColumnChartSpecModel(- mockData,+ [],specialFieldTypes,specialStats,- mockBinValues,+ specialBinValues,mockValueCounts,{ includeCharts: true },);@@ -139,17 +146,17 @@ describe("ColumnChartSpecModel", () => {expect(// @ts-expect-error layer should be available(summary.spec?.layer[0].encoding?.x as { field: string })?.field,- ).toBe("column\\.with\\[special\\:chars\\]");+ ).toBe("bin_start");});- it("should expect bin values to be used for number and integer columns when feat flag is true", () => {+ it("should expect bin values to be used for number and integer columns", () => {const model = new ColumnChartSpecModel(- mockData,+ [],mockFieldTypes,mockStats,mockBinValues,mockValueCounts,- { includeCharts: true, usePreComputedValues: true },+ { includeCharts: true },);const summary = model.getHeaderSummary("number");expect(summary.spec).toBeDefined();@@ -171,14 +178,14 @@ describe("ColumnChartSpecModel", () => {expect(summary2.spec?.data?.values).toEqual(mockBinValues.integer);});- it("should handle datetime bin values when feat flag is true", () => {+ it("should handle datetime bin values", () => {const model = new ColumnChartSpecModel(- mockData,+ [],mockFieldTypes,mockStats,mockBinValues,mockValueCounts,- { includeCharts: true, usePreComputedValues: true },+ { includeCharts: true },);const summary = model.getHeaderSummary("datetime");@@ -186,9 +193,6 @@ describe("ColumnChartSpecModel", () => {// @ts-expect-error data.values should be availableexpect(summary.spec?.data?.values).toEqual(mockBinValues.datetime);- // Expect hconcat since there are nulls- // @ts-expect-error hconcat should be available- expect(summary.spec?.hconcat).toBeDefined();expect(summary.spec).toMatchSnapshot();// Test again without the nulls@@ -197,12 +201,12 @@ describe("ColumnChartSpecModel", () => {datetime: { min: "2023-01-01", max: "2023-12-31" },};const model2 = new ColumnChartSpecModel(- mockData,+ [],mockFieldTypes,mockStats2,mockBinValues,mockValueCounts,- { includeCharts: true, usePreComputedValues: true },+ { includeCharts: true },);const summary2 = model2.getHeaderSummary("datetime");expect(summary2.spec).toBeDefined();@@ -212,14 +216,14 @@ describe("ColumnChartSpecModel", () => {expect(summary2.spec?.hconcat).toBeUndefined();});- it("should handle boolean stats when feat flag is true", () => {+ it("should handle boolean stats", () => {const model = new ColumnChartSpecModel(- mockData,+ [],mockFieldTypes,mockStats,mockBinValues,mockValueCounts,- { includeCharts: true, usePreComputedValues: true },+ { includeCharts: true },);const summary = model.getHeaderSummary("boolean");expect(summary.spec).toBeDefined();@@ -234,14 +238,14 @@ describe("ColumnChartSpecModel", () => {expect(summary.spec).toMatchSnapshot();});- it("should handle string value counts when feat flag is true", () => {+ it("should handle string value counts", () => {const model = new ColumnChartSpecModel(- mockData,+ [],mockFieldTypes,mockStats,mockBinValues,mockValueCounts,- { includeCharts: true, usePreComputedValues: true },+ { includeCharts: true },);const summary = model.getHeaderSummary("string");expect(summary.spec).toBeDefined();@@ -262,7 +266,7 @@ describe("ColumnChartSpecModel", () => {expect(summary.spec).toMatchSnapshot();});- describe("snapshot", () => {+ describe("snapshot with legacy data spec", () => {const fieldTypes: FieldTypes = {...mockFieldTypes,a: "number",diff --git a/frontend/src/components/data-table/column-summary/chart-spec-model.tsx b/frontend/src/components/data-table/column-summary/chart-spec-model.tsxindex d4b4ebbebc..98d86a633d 100644--- a/frontend/src/components/data-table/column-summary/chart-spec-model.tsx+++ b/frontend/src/components/data-table/column-summary/chart-spec-model.tsx@@ -3,19 +3,8 @@import { mint, orange, slate } from "@radix-ui/colors";import type { TopLevelSpec } from "vega-lite";import type { StringFieldDef } from "vega-lite/build/src/channeldef";-// @ts-expect-error vega-typings does not include formats-import { formats } from "vega-loader";-import { asRemoteURL } from "@/core/runtime/config";import type { TopLevelFacetedUnitSpec } from "@/plugins/impl/data-explorer/queries/types";-import { arrow } from "@/plugins/impl/vega/formats";-import { parseCsvData } from "@/plugins/impl/vega/loader";import { logNever } from "@/utils/assertNever";-import {- byteStringToBinary,- extractBase64FromDataURL,- isDataURLString,- typedAtob,-} from "@/utils/json/base64";import type {BinValues,ColumnHeaderStats,@@ -24,9 +13,11 @@ import type {ValueCounts,} from "../types";import {+ getDataSpecAndSourceName,getLegacyBooleanSpec,getLegacyNumericSpec,getLegacyTemporalSpec,+ getScale,} from "./legacy-chart-spec";import { calculateBinStep, getPartialTimeTooltip } from "./utils";@@ -34,15 +25,13 @@ import { calculateBinStep, getPartialTimeTooltip } from "./utils";const MAX_BAR_HEIGHT = 20; // px// If we are concatenating charts, we need to specify each chart's height and width.-const CHART_HEIGHT = 30;-const CHART_WIDTH = 70;-const NULL_BAR_WIDTH = 5;--// Arrow formats have a magic number at the beginning of the file.-const ARROW_MAGIC_NUMBER = "ARROW1";+const CONCAT_CHART_HEIGHT = 30;+const CONCAT_CHART_WIDTH = 70;+const CONCAT_NULL_BAR_WIDTH = 5;-// register arrow reader under type 'arrow'-formats("arrow", arrow);+const BAR_COLOR = mint.mint11;+const UNHOVERED_BAR_OPACITY = 0.6;+const NULL_BAR_COLOR = orange.orange11;export class ColumnChartSpecModel<T> {private columnStats = new Map<ColumnName, Partial<ColumnHeaderStats>>();@@ -57,21 +46,21 @@ export class ColumnChartSpecModel<T> {{},{includeCharts: false,- usePreComputedValues: false,},);private dataSpec: TopLevelSpec["data"];- private sourceName: "data_0" | "source_0";- private readonly data: T[] | string;+ // Legacy data spec for fallback+ private legacyDataSpec: TopLevelSpec["data"];+ private legacySourceName: "data_0" | "source_0";+private readonly fieldTypes: FieldTypes;readonly stats: Record<ColumnName, Partial<ColumnHeaderStats>>;readonly binValues: Record<ColumnName, BinValues>;readonly valueCounts: Record<ColumnName, ValueCounts>;private readonly opts: {includeCharts: boolean;- usePreComputedValues?: boolean;};constructor(@@ -82,56 +71,22 @@ export class ColumnChartSpecModel<T> {valueCounts: Record<ColumnName, ValueCounts>,opts: {includeCharts: boolean;- usePreComputedValues?: boolean;},) {- this.data = data;this.fieldTypes = fieldTypes;this.stats = stats;this.binValues = binValues;this.valueCounts = valueCounts;this.opts = opts;- // Data may come in from a few different sources:- // - A URL- // - A CSV data URI (e.g. "data:text/csv;base64,...")- // - A CSV string (e.g. "a,b,c\n1,2,3\n4,5,6")- // - An array of objects- // For each case, we need to set up the data spec and source name appropriately.- // If its a file, the source name will be "source_0", otherwise it will be "data_0".- // We have a few snapshot tests to ensure that the spec is correct for each case.- if (typeof this.data === "string") {- if (this.data.startsWith("./@file") || this.data.startsWith("/@file")) {- this.dataSpec = { url: asRemoteURL(this.data).href };- this.sourceName = "source_0";- } else if (isDataURLString(this.data)) {- this.sourceName = "data_0";- const base64 = extractBase64FromDataURL(this.data);- const decoded = typedAtob(base64);-- if (decoded.startsWith(ARROW_MAGIC_NUMBER)) {- // @ts-expect-error vega-typings does not include arrow format- this.dataSpec = {- values: byteStringToBinary(decoded),- format: { type: "arrow" },- };- } else {- // Assume it's a CSV string- this.parseCsv(decoded);- }- } else {- // Assume it's a CSV string- this.parseCsv(this.data);- this.sourceName = "data_0";- }- } else {- this.dataSpec = { values: this.data };- this.sourceName = "source_0";- }-this.columnBinValues = new Map(Object.entries(binValues));this.columnValueCounts = new Map(Object.entries(valueCounts));this.columnStats = new Map(Object.entries(stats));++ const { dataSpec, sourceName } = getDataSpecAndSourceName(data);+ this.dataSpec = dataSpec;+ this.legacyDataSpec = dataSpec;+ this.legacySourceName = sourceName;}public getColumnStats(column: string) {@@ -146,42 +101,8 @@ export class ColumnChartSpecModel<T> {};}- private parseCsv(data: string) {- this.dataSpec = {- values: parseCsvData(data) as T[],- };- }-- private getVegaSpec(column: string): TopLevelFacetedUnitSpec | null {- if (!this.data) {- return null;- }-- const usePreComputedValues = this.opts.usePreComputedValues;- const binValues = this.columnBinValues.get(column);- const valueCounts = this.columnValueCounts.get(column);- const hasValueCounts = valueCounts && valueCounts.length > 0;-- let data = this.dataSpec as TopLevelFacetedUnitSpec["data"];- const stats = this.columnStats.get(column);-- if (usePreComputedValues) {- if (hasValueCounts) {- data = { values: valueCounts, name: "value_counts" };- } else {- // Bin values can be empty if all values are nulls- if (stats?.nulls) {- binValues?.push({- bin_start: null,- bin_end: null,- count: stats.nulls as number,- });- }- data = { values: binValues, name: "bin_values" };- }- }-- const base: TopLevelFacetedUnitSpec = {+ private createBase(data: TopLevelSpec["data"]): TopLevelFacetedUnitSpec {+ return {background: "transparent",data,config: {@@ -194,7 +115,31 @@ export class ColumnChartSpecModel<T> {},height: 100,} as TopLevelFacetedUnitSpec;+ }++ private getVegaSpec(column: string): TopLevelFacetedUnitSpec | null {+ const binValues = this.columnBinValues.get(column);+ const valueCounts = this.columnValueCounts.get(column);+ const hasValueCounts = valueCounts && valueCounts.length > 0;++ let data = this.dataSpec as TopLevelFacetedUnitSpec["data"];+ const stats = this.columnStats.get(column);+ if (hasValueCounts) {+ data = { values: valueCounts, name: "value_counts" };+ } else {+ // Bin values can be empty if all values are nulls+ if (stats?.nulls) {+ binValues?.push({+ bin_start: null,+ bin_end: null,+ count: stats.nulls as number,+ });+ }+ data = { values: binValues, name: "bin_values" };+ }++ const base = this.createBase(data);const type = this.fieldTypes[column];// https://github.com/vega/altair/blob/32990a597af7c09586904f40b3f5e6787f752fa5/doc/user_guide/encodings/index.rst#escaping-special-characters-in-column-names@@ -205,18 +150,16 @@ export class ColumnChartSpecModel<T> {// escape colons in column namescolumn = column.replaceAll(":", "\\:");- const scale = this.getScale();-switch (type) {case "date":case "datetime":case "time": {- if (!usePreComputedValues) {- return getLegacyTemporalSpec(column, type, base, scale);+ if (!binValues) {+ const legacyBase = this.createBase(this.legacyDataSpec);+ const scale = getScale(this.legacySourceName);+ return getLegacyTemporalSpec(column, type, legacyBase, scale);}- const binStep = calculateBinStep(binValues || []);-const tooltip = getPartialTimeTooltip(binValues || []);const singleValue = binValues?.length === 1;@@ -224,7 +167,7 @@ export class ColumnChartSpecModel<T> {if (singleValue) {return {...base,- mark: { type: "bar", color: mint.mint11 },+ mark: { type: "bar", color: BAR_COLOR },encoding: {x: {field: "bin_start",@@ -255,16 +198,12 @@ export class ColumnChartSpecModel<T> {}const histogram: TopLevelFacetedUnitSpec = {- height: CHART_HEIGHT,- width: CHART_WIDTH,// @ts-expect-error 'layer' property not in TopLevelFacetedUnitSpeclayer: [{mark: {type: "bar",- color: mint.mint11,- stroke: mint.mint11,- strokeWidth: 0,+ color: BAR_COLOR,},params: [{@@ -279,13 +218,7 @@ export class ColumnChartSpecModel<T> {encoding: {x: {field: "bin_start",- type: "temporal",- bin: { binned: true, step: binStep },- axis: null,- },- x2: {- field: "bin_end",- type: "temporal",+ type: "ordinal",axis: null,},y: {@@ -293,13 +226,21 @@ export class ColumnChartSpecModel<T> {type: "quantitative",axis: null,},- strokeWidth: {+ color: {condition: {- param: "hover",- empty: false,- value: 0.5,+ test: "datum['bin_start'] === null && datum['bin_end'] === null",+ value: NULL_BAR_COLOR,},- value: 0,+ value: BAR_COLOR,+ },+ opacity: {+ condition: [+ {+ param: "hover",+ value: 1,+ },+ ],+ value: UNHOVERED_BAR_OPACITY,},},},@@ -309,18 +250,13 @@ export class ColumnChartSpecModel<T> {mark: {type: "bar",opacity: 0,+ // Wider bars to cover gaps between bars, prevents flickering when hovering over bars+ width: { band: 1.2 },},encoding: {x: {field: "bin_start",- type: "temporal",- bin: { binned: true, step: binStep },- axis: null,- },- x2: {- field: "bin_end",- type: "temporal",- bin: { binned: true, step: binStep },+ type: "ordinal",axis: null,},y: {@@ -353,130 +289,33 @@ export class ColumnChartSpecModel<T> {],};- const nullBar: TopLevelFacetedUnitSpec = {- height: CHART_HEIGHT,- width: NULL_BAR_WIDTH,- // @ts-expect-error 'layer' property not in TopLevelFacetedUnitSpec- layer: [- {- mark: {- type: "bar",- color: orange.orange11,- },- encoding: {- x: {- field: "bin_start",- type: "nominal",- axis: null,- },- y: {- field: "count",- type: "quantitative",- axis: null,- },- },- },-- // Invisible tooltip layer with max-height- {- mark: {- type: "bar",- opacity: 0,- },- encoding: {- x: {- field: "bin_start",- type: "nominal",- axis: null,- },- y: {- aggregate: "max",- type: "quantitative",- axis: null,- },- tooltip: [- {- field: "count",- type: "quantitative",- title: "nulls",- format: ",d",- },- ],- },- },- ],- transform: [- {- filter:- "datum['bin_start'] === null && datum['bin_end'] === null",- },- ],- };-- let chart: TopLevelFacetedUnitSpec = histogram;- let timeBase = base;-- if (stats?.nulls) {- timeBase = {- ...base,- config: {- ...base.config,- concat: {- spacing: 0,- },- },- resolve: {- scale: {- y: "shared",- },- },- };- chart = {- // Temporal axis will not show nulls, so we concat 2 charts- // @ts-expect-error 'hconcat' property not in TopLevelFacetedUnitSpec- hconcat: [nullBar, histogram],- };- }-return {- ...timeBase,- ...chart,+ ...base,+ ...histogram,};}case "integer":case "number": {// Create a histogram spec that properly handles null valuesconst format = type === "integer" ? ",d" : ".2f";- const binStep = calculateBinStep(binValues || []);- if (!usePreComputedValues) {- return getLegacyNumericSpec(column, format, base);+ if (!binValues) {+ const legacyBase = this.createBase(this.legacyDataSpec);+ return getLegacyNumericSpec(column, format, legacyBase);}- const stats = this.columnStats.get(column);+ const binStep = calculateBinStep(binValues || []);const histogram: TopLevelFacetedUnitSpec = {- height: CHART_HEIGHT,- width: CHART_WIDTH,+ height: CONCAT_CHART_HEIGHT,+ width: CONCAT_CHART_WIDTH,// @ts-expect-error 'layer' property not in TopLevelFacetedUnitSpeclayer: [{mark: {type: "bar",- color: mint.mint11,- stroke: mint.mint11,- strokeWidth: 0,+ color: BAR_COLOR,},- params: [- {- name: "hover",- select: {- type: "point",- on: "mouseover",- clear: "mouseout",- },- },- ],encoding: {x: {field: "bin_start",@@ -492,13 +331,14 @@ export class ColumnChartSpecModel<T> {type: "quantitative",axis: null,},- strokeWidth: {- condition: {- param: "hover",- empty: false,- value: 0.5,- },- value: 0,+ opacity: {+ condition: [+ {+ param: "hover",+ value: 1,+ },+ ],+ value: UNHOVERED_BAR_OPACITY,},},},@@ -509,6 +349,17 @@ export class ColumnChartSpecModel<T> {type: "bar",opacity: 0,},+ params: [+ {+ name: "hover",+ select: {+ type: "point",+ on: "mouseover",+ clear: "mouseout",+ nearest: true, // Nearest avoids flickering when hovering over bars, but it's not perfect+ },+ },+ ],encoding: {x: {field: "bin_start",@@ -564,14 +415,14 @@ export class ColumnChartSpecModel<T> {};const nullBar: TopLevelFacetedUnitSpec = {- height: CHART_HEIGHT,- width: NULL_BAR_WIDTH,+ height: CONCAT_CHART_HEIGHT,+ width: CONCAT_NULL_BAR_WIDTH,// @ts-expect-error 'layer' property not in TopLevelFacetedUnitSpeclayer: [{mark: {type: "bar",- color: orange.orange11,+ color: NULL_BAR_COLOR,},encoding: {x: {@@ -652,7 +503,7 @@ export class ColumnChartSpecModel<T> {};}case "boolean": {- if (!usePreComputedValues || !stats?.true || !stats?.false) {+ if (!stats?.true || !stats?.false) {return getLegacyBooleanSpec(column, base, MAX_BAR_HEIGHT);}@@ -699,7 +550,7 @@ export class ColumnChartSpecModel<T> {},mark: {type: "bar",- color: mint.mint11,+ color: BAR_COLOR,},encoding: {y: {@@ -726,7 +577,7 @@ export class ColumnChartSpecModel<T> {type: "nominal",scale: {domain: ["true", "false", "null"],- range: [mint.mint11, mint.mint11, orange.orange11],+ range: [BAR_COLOR, BAR_COLOR, NULL_BAR_COLOR],},legend: null,},@@ -740,7 +591,7 @@ export class ColumnChartSpecModel<T> {{mark: {type: "bar",- color: mint.mint11,+ color: BAR_COLOR,height: BAR_HEIGHT,},},@@ -764,7 +615,7 @@ export class ColumnChartSpecModel<T> {} as TopLevelFacetedUnitSpec; // "layer" not in TopLevelFacetedUnitSpec}case "string": {- if (!usePreComputedValues || !hasValueCounts) {+ if (!hasValueCounts) {return null;}@@ -840,18 +691,20 @@ export class ColumnChartSpecModel<T> {type: "quantitative",},color: {+ condition: {+ test: `datum.${yField} == "None" || datum.${yField} == "null"`,+ value: NULL_BAR_COLOR,+ },+ value: BAR_COLOR,+ },+ opacity: {condition: [{param: "hover_bar",- value: mint.mint11,- },- {- test: `datum.${yField} == "None" || datum.${yField} == "null"`,- value: orange.orange11,+ value: 1,},],- value: mint.mint8,- legend: null,+ value: UNHOVERED_BAR_OPACITY,},tooltip: [{@@ -920,14 +773,4 @@ export class ColumnChartSpecModel<T> {return null;}}-- private getScale() {- return {- align: 0,- paddingInner: 0,- paddingOuter: {- expr: `length(data('${this.sourceName}')) == 2 ? 1 : length(data('${this.sourceName}')) == 3 ? 0.5 : length(data('${this.sourceName}')) == 4 ? 0 : 0`,- },- };- }}diff --git a/frontend/src/components/data-table/column-summary/column-summary.tsx b/frontend/src/components/data-table/column-summary/column-summary.tsxindex d8a5abbd79..737c772a92 100644--- a/frontend/src/components/data-table/column-summary/column-summary.tsx+++ b/frontend/src/components/data-table/column-summary/column-summary.tsx@@ -48,7 +48,7 @@ export const TableColumnSummary = <TData, TValue>({<Suspense fallback={skeleton}><LazyVegaLitespec={spec}- width={70}+ width={80}height={30}renderer="svg"// @ts-expect-error - Our `loader.load` method is broader than VegaLite's typings but is functionally supported.diff --git a/frontend/src/components/data-table/column-summary/legacy-chart-spec.ts b/frontend/src/components/data-table/column-summary/legacy-chart-spec.tsindex 9dbdafaff0..63e1e50321 100644--- a/frontend/src/components/data-table/column-summary/legacy-chart-spec.ts+++ b/frontend/src/components/data-table/column-summary/legacy-chart-spec.ts@@ -1,10 +1,22 @@/* Copyright 2024 Marimo. All rights reserved. */-// New spec is under a feature flag, we keep this until full migration+// Fallback specimport { mint, orange, slate } from "@radix-ui/colors";+import type { TopLevelSpec } from "vega-lite";import type { Scale } from "vega-lite/build/src/scale";+// @ts-expect-error vega-typings does not include formats+import { formats } from "vega-loader";+import { asRemoteURL } from "@/core/runtime/config";import type { TopLevelFacetedUnitSpec } from "@/plugins/impl/data-explorer/queries/types";+import { arrow } from "@/plugins/impl/vega/formats";+import { parseCsvData } from "@/plugins/impl/vega/loader";+import {+ byteStringToBinary,+ extractBase64FromDataURL,+ isDataURLString,+ typedAtob,+} from "@/utils/json/base64";export function getLegacyNumericSpec(column: string,@@ -240,3 +252,67 @@ export function getLegacyBooleanSpec(],} as TopLevelFacetedUnitSpec; // "layer" not in TopLevelFacetedUnitSpec}++// Arrow formats have a magic number at the beginning of the file.+const ARROW_MAGIC_NUMBER = "ARROW1";++// register arrow reader under type 'arrow'+formats("arrow", arrow);++export function getDataSpecAndSourceName<T>(data: string | T[]): {+ dataSpec: TopLevelSpec["data"];+ sourceName: "data_0" | "source_0";+} {+ let dataSpec: TopLevelSpec["data"];+ let sourceName: "data_0" | "source_0";++ // Data may come in from a few different sources:+ // - A URL+ // - A CSV data URI (e.g. "data:text/csv;base64,...")+ // - A CSV string (e.g. "a,b,c\n1,2,3\n4,5,6")+ // - An array of objects+ // For each case, we need to set up the data spec and source name appropriately.+ // If its a file, the source name will be "source_0", otherwise it will be "data_0".+ // We have a few snapshot tests to ensure that the spec is correct for each case.+ if (typeof data === "string") {+ if (data.startsWith("./@file") || data.startsWith("/@file")) {+ dataSpec = { url: asRemoteURL(data).href };+ sourceName = "source_0";+ } else if (isDataURLString(data)) {+ sourceName = "data_0";+ const base64 = extractBase64FromDataURL(data);+ const decoded = typedAtob(base64);++ // eslint-disable-next-line unicorn/prefer-ternary+ if (decoded.startsWith(ARROW_MAGIC_NUMBER)) {+ // @ts-expect-error vega-typings does not include arrow format+ dataSpec = {+ values: byteStringToBinary(decoded),+ format: { type: "arrow" },+ };+ } else {+ // Assume it's a CSV string+ dataSpec = { values: parseCsvData(decoded) };+ }+ } else {+ // Assume it's a CSV string+ dataSpec = { values: parseCsvData(data) };+ sourceName = "data_0";+ }+ } else {+ dataSpec = { values: data };+ sourceName = "source_0";+ }++ return { dataSpec, sourceName };+}++export function getScale(sourceName: string): Scale {+ return {+ align: 0,+ paddingInner: 0,+ paddingOuter: {+ expr: `length(data('${sourceName}')) == 2 ? 1 : length(data('${sourceName}')) == 3 ? 0.5 : length(data('${sourceName}')) == 4 ? 0 : 0`,+ },+ };+}diff --git a/frontend/src/plugins/impl/DataTablePlugin.tsx b/frontend/src/plugins/impl/DataTablePlugin.tsxindex 607008665f..0922558c92 100644--- a/frontend/src/plugins/impl/DataTablePlugin.tsx+++ b/frontend/src/plugins/impl/DataTablePlugin.tsx@@ -57,7 +57,6 @@ import { Alert, AlertTitle } from "@/components/ui/alert";import { Button } from "@/components/ui/button";import { DelayMount } from "@/components/utils/delay-mount";import { type CellId, findCellId } from "@/core/cells/ids";-import { getFeatureFlag } from "@/core/config/feature-flag";import { slotsController } from "@/core/slots/slots";import { store } from "@/core/state/jotai";import { isStaticNotebook } from "@/core/static/static-state";@@ -85,15 +84,12 @@ import {type CsvURL = string;export type TableData<T> = T[] | CsvURL;-interface ColumnSummariesArgs {- precompute: boolean;-}-interface ColumnSummaries<T = unknown> {data: TableData<T> | null | undefined;stats: Record<ColumnName, ColumnHeaderStats>;bin_values: Record<ColumnName, BinValues>;value_counts: Record<ColumnName, ValueCounts>;+ show_charts: boolean;is_disabled?: boolean;}@@ -198,9 +194,7 @@ interface Data<T> {// eslint-disable-next-line @typescript-eslint/consistent-type-definitionstype DataTableFunctions = {download_as: DownloadAsArgs;- get_column_summaries: <T>(- opts: ColumnSummariesArgs,- ) => Promise<ColumnSummaries<T>>;+ get_column_summaries: <T>(opts: {}) => Promise<ColumnSummaries<T>>;search: <T>(req: {sort?: {by: string;@@ -281,19 +275,16 @@ export const DataTablePlugin = createPlugin<S>("marimo-table")).withFunctions<DataTableFunctions>({download_as: DownloadAsSchema,- get_column_summaries: rpc- .input(z.object({ precompute: z.boolean() }))- .output(- z.object({- data: z- .union([z.string(), z.array(z.object({}).passthrough())])- .nullable(),- stats: z.record(z.string(), columnStats),- bin_values: z.record(z.string(), binValues),- value_counts: z.record(z.string(), valueCounts),- is_disabled: z.boolean().optional(),- }),- ),+ get_column_summaries: rpc.input(z.looseObject({})).output(+ z.object({+ data: z.union([z.string(), z.array(z.looseObject({}))]).nullable(),+ stats: z.record(z.string(), columnStats),+ bin_values: z.record(z.string(), binValues),+ value_counts: z.record(z.string(), valueCounts),+ show_charts: z.boolean(),+ is_disabled: z.boolean().optional(),+ }),+ ),search: rpc.input(z.object({@@ -598,8 +589,6 @@ export const LoadingDataTableComponent = memo();}, [data?.totalRows]);- const precompute = getFeatureFlag("performant_table_charts");-// Column summariesconst { data: columnSummaries, error: columnSummariesError } = useAsyncData<ColumnSummaries<T>@@ -607,9 +596,15 @@ export const LoadingDataTableComponent = memo(// TODO: props.get_column_summaries is always true,// so we are unable to detect if the function is registeredif (props.totalRows === 0 || !props.showColumnSummaries) {- return { data: null, stats: {}, bin_values: {}, value_counts: {} };+ return {+ data: null,+ stats: {},+ bin_values: {},+ value_counts: {},+ show_charts: false,+ };}- return props.get_column_summaries({ precompute });+ return props.get_column_summaries({});}, [props.get_column_summaries,props.showColumnSummaries,@@ -778,8 +773,7 @@ const DataTableComponent = ({columnSummaries.bin_values,columnSummaries.value_counts,{- includeCharts: Boolean(columnSummaries.data),- usePreComputedValues: getFeatureFlag("performant_table_charts"),+ includeCharts: columnSummaries.show_charts,},);}, [fieldTypes, columnSummaries]);diff --git a/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx b/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsxindex e0a2f1d43b..b1767f3e7c 100644--- a/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx+++ b/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx@@ -312,9 +312,10 @@ DataFrameComponent.displayName = "DataFrameComponent";function getColumnSummaries() {return Promise.resolve({- stats: {},data: null,+ stats: {},bin_values: {},value_counts: {},+ show_charts: false,});}diff --git a/marimo/_plugins/ui/_impl/table.py b/marimo/_plugins/ui/_impl/table.pyindex fc153c565c..eefeb4ecc4 100644--- a/marimo/_plugins/ui/_impl/table.py+++ b/marimo/_plugins/ui/_impl/table.py@@ -89,24 +89,24 @@ class DownloadAsArgs:@dataclass-class ColumnSummariesArgs:- """If enabled, we will precompute chart values."""-- precompute: bool+class ColumnSummariesArgs: ...@dataclassclass ColumnSummaries:+ # If precomputed aggregations fail, we fallback to chart datadata: Union[JSONType, str]stats: dict[ColumnName, ColumnStats]bin_values: dict[ColumnName, list[BinValue]]value_counts: dict[ColumnName, list[ValueCount]]+ show_charts: bool# Disabled because of too many columns/rows# This will show a banner in the frontendis_disabled: Optional[bool] = NoneShowColumnSummaries = Union[bool, Literal["stats", "chart"]]+CHART_MAX_ROWS_STRING_VALUE_COUNTS = 20_000DEFAULT_MAX_COLUMNS = 50@@ -888,6 +888,8 @@ class table(If summaries are disabled or row limit is exceeded, returns emptysummaries with is_disabled flag set appropriately."""+ del args+show_column_summaries = self._show_column_summariesif not show_column_summaries:@@ -899,6 +901,7 @@ class table(# This is not 'disabled' because of too many rows# so we don't want to display the banneris_disabled=False,+ show_charts=False,)total_rows = self._searched_manager.get_num_rows(force=True) or 0@@ -912,12 +915,13 @@ class table(bin_values={},value_counts={},is_disabled=True,+ show_charts=False,)# If we are above the limit to show charts,# or if we are in stats-only mode,- # we don't return the chart data- should_get_chart_data = (+ # we don't show charts+ show_charts = (self._show_column_summaries != "stats"and total_rows <= self._column_charts_row_limit)@@ -934,6 +938,9 @@ class table(DEFAULT_BIN_SIZE = 9DEFAULT_VALUE_COUNTS_SIZE = 15+ bin_aggregation_failed = False+ cols_to_drop = []+for column in self._manager.get_column_names():statistic = Noneif should_get_stats:@@ -945,25 +952,25 @@ class table(# BaseExceptions, which shouldn't crash the kernelLOGGER.warning("Failed to get stats for column %s", column)- if should_get_chart_data and args.precompute:+ if show_charts:if not should_get_stats:LOGGER.warning("Unable to compute stats for column, may not be computed correctly")- # For boolean columns, we can drop the column since we use stats(column_type, external_type) = self._manager.get_field_type(column)- if column_type == "boolean":- data = data.drop_columns([column])+ # For boolean columns, we can drop the column since we use stats+ if column_type == "boolean" or column_type == "unknown":+ cols_to_drop.append(column)# Handle columns with all nulls first# These get empty bins regardless of typeif statistic and statistic.nulls == total_rows:try:bin_values[column] = []- data = data.drop_columns([column])+ cols_to_drop.append(column)continueexcept BaseException as e:LOGGER.warning(@@ -971,10 +978,15 @@ class table()continue- # For perf, we only compute value counts for categorical columns+ # For now, we only compute value counts for categorical columns and small tablesexternal_type = external_type.lower()- if column_type == "string" and (- "cat" in external_type or "enum" in external_type+ if (+ column_type == "string"+ and ("cat" in external_type or "enum" in external_type)+ or (+ column_type == "string"+ and total_rows <= CHART_MAX_ROWS_STRING_VALUE_COUNTS+ )):try:val_counts = self._get_value_counts(@@ -982,7 +994,7 @@ class table()if len(val_counts) > 0:value_counts[column] = val_counts- data = data.drop_columns([column])+ cols_to_drop.append(column)continueexcept BaseException as e:LOGGER.warning(@@ -1002,18 +1014,23 @@ class table(continuetry:+ # get_bin_values is marked unstable+ # https://narwhals-dev.github.io/narwhals/api-reference/series/#narwhals.series.Series.histbins = data.get_bin_values(column, DEFAULT_BIN_SIZE)bin_values[column] = bins- # Only drop column if we got bins to visualizeif len(bins) > 0:- data = data.drop_columns([column])+ cols_to_drop.append(column)continueexcept BaseException as e:+ bin_aggregation_failed = TrueLOGGER.warning("Failed to get bin values for column %s: %s", column, e)- if should_get_chart_data:+ should_fallback = show_charts and bin_aggregation_failed+ if should_fallback:+ LOGGER.debug("Bin aggregation failed, falling back to chart data")+ data = data.drop_columns(cols_to_drop)chart_data, _ = self._to_chart_data_url(data)return ColumnSummaries(@@ -1021,6 +1038,7 @@ class table(stats=stats,bin_values=bin_values,value_counts=value_counts,+ show_charts=show_charts,is_disabled=False,)diff --git a/marimo/_smoke_tests/tables/column-header-chart.py b/marimo/_smoke_tests/tables/column-header-chart.pynew file mode 100644index 0000000000..d18b8d14a3--- /dev/null+++ b/marimo/_smoke_tests/tables/column-header-chart.py@@ -0,0 +1,72 @@+# /// script+# requires-python = ">=3.13"+# dependencies = [+# "polars==1.34.0",+# "requests==2.32.5",+# ]+# ///++import marimo++__generated_with = "0.17.0"+app = marimo.App(width="medium")+++def _():+ import marimo as mo+ import polars as pl+ return (pl,)+++def _(pl):+ # Enums and categorical data types+ bears_enum = pl.Enum(["Polar", "Panda", "Brown"])+ bears = pl.Series(+ ["Polar", "Panda", "Brown", "Brown", "Polar"] * 30, dtype=bears_enum+ )++ enums_cats = pl.DataFrame(+ {+ "bears": ["Polar", "Panda", "Brown", "Brown", "Polar"] * 30,+ "bears_cat": ["Polar", "Panda", "Brown", "Brown", "Polar"] * 30,+ },+ schema={+ "bears": bears_enum,+ "bears_cat": pl.Categorical,+ },+ )+ enums_cats+ return+++def _(pl):+ pokemon_url = "https://gist.githubusercontent.com/armgilles/194bcff35001e7eb53a2a8b441e8b2c6/raw/92200bc0a673d5ce2110aaad4544ed6c4010f687/pokemon.csv"+ pl.read_csv(pokemon_url)+ return+++def _(pl):+ import io+ import requests+ import zipfile++ train_parquet_link = "https://www.kaggle.com/api/v1/datasets/download/shahmirvarqha/train-stations-amsterdam"++ response = requests.get(train_parquet_link)+ zip_data = io.BytesIO(response.content)+ with zipfile.ZipFile(zip_data) as z:+ # List files to find parquet file, assuming one parquet file is in the archive+ parquet_file_name = [f for f in z.namelist() if f.endswith(".parquet")][0]+ with z.open(parquet_file_name) as parquet_file:+ trains_df = pl.read_parquet(parquet_file)++ trains_df[:20000]+ return+++if __name__ == "__main__":+ app.run()diff --git a/marimo/_smoke_tests/tables/complex_types.py b/marimo/_smoke_tests/tables/complex_types.pyindex 6ae6c7a6c8..83975e52d0 100644--- a/marimo/_smoke_tests/tables/complex_types.py+++ b/marimo/_smoke_tests/tables/complex_types.py@@ -1,6 +1,6 @@import marimo-__generated_with = "0.16.5"+__generated_with = "0.17.2"app = marimo.App(width="medium")@@ -97,6 +97,9 @@ def _():),})++ # df = pl.concat([df]*10, rechunk=True).with_row_count(name="idx")+mo.ui.table(df)return df, mo, pl@@ -193,5 +196,6 @@ def _():return+if __name__ == "__main__":app.run()diff --git a/pixi.lock b/pixi.lockindex 74ea5e1fa9..83b5dadd43 100644--- a/pixi.lock+++ b/pixi.lock@@ -1101,8 +1101,8 @@ packages:timestamp: 1727801725384- pypi: ./name: marimo- version: 0.17.0- sha256: 5e2b68e9ec635e489b7e57183e46a5199d745621494165518f589a5498327212+ version: 0.17.2+ sha256: d81b66fd7b9230f46e084209d14ea657198fa4a86285c07a2e1956d761d985cdrequires_dist:- click>=8.0,<9- jedi>=0.18.0diff --git a/pyproject.toml b/pyproject.tomlindex 8d080e2118..33d9bd1d88 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -549,7 +549,6 @@ format_on_save = true[tool.marimo.experimental]multi_column = true-performant_table_charts = truechat_modes = truecache_panel = truediff --git a/tests/_plugins/ui/_impl/test_table.py b/tests/_plugins/ui/_impl/test_table.pyindex 439e964238..3190578def 100644--- a/tests/_plugins/ui/_impl/test_table.py+++ b/tests/_plugins/ui/_impl/test_table.py@@ -13,6 +13,7 @@ from marimo._dependencies.dependencies import DependencyManagerfrom marimo._plugins import uifrom marimo._plugins.ui._impl.dataframes.transforms.types import Conditionfrom marimo._plugins.ui._impl.table import (+ CHART_MAX_ROWS_STRING_VALUE_COUNTS,DEFAULT_MAX_COLUMNS,MAX_COLUMNS_NOT_PROVIDED,CalculateTopKRowsArgs,@@ -849,9 +850,7 @@ def test_table_with_too_many_rows_column_summaries_disabled() -> None:data = {"a": list(range(20))}table = ui.table(data, _internal_summary_row_limit=10)- summaries_disabled = table._get_column_summaries(- ColumnSummariesArgs(precompute=False)- )+ summaries_disabled = table._get_column_summaries(ColumnSummariesArgs())assert summaries_disabled.is_disabled is True# search results are 2 and 12@@ -862,9 +861,7 @@ def test_table_with_too_many_rows_column_summaries_disabled() -> None:page_number=0,))- summaries_enabled = table._get_column_summaries(- ColumnSummariesArgs(precompute=False)- )+ summaries_enabled = table._get_column_summaries(ColumnSummariesArgs())assert summaries_enabled.is_disabled is False@@ -872,11 +869,9 @@ def test_with_too_many_rows_column_charts_disabled() -> None:data = {"a": list(range(20))}table = ui.table(data, _internal_column_charts_row_limit=10)- charts_disabled = table._get_column_summaries(- ColumnSummariesArgs(precompute=False)- )+ charts_disabled = table._get_column_summaries(ColumnSummariesArgs())+ assert charts_disabled.show_charts is Falseassert charts_disabled.is_disabled is False- assert charts_disabled.data is None# search results are 2 and 12table._search(@@ -886,9 +881,9 @@ def test_with_too_many_rows_column_charts_disabled() -> None:page_number=0,))- charts_enabled = table._get_column_summaries(- ColumnSummariesArgs(precompute=False)- )+ charts_enabled = table._get_column_summaries(ColumnSummariesArgs())+ assert charts_enabled.show_charts is True+ assert charts_enabled.data is Noneassert charts_enabled.is_disabled is False@@ -905,13 +900,9 @@ def test_get_column_summaries_after_search() -> None:page_number=0,))- summaries = table._get_column_summaries(- ColumnSummariesArgs(precompute=False)- )+ summaries = table._get_column_summaries(ColumnSummariesArgs())+ assert summaries.show_charts is Trueassert summaries.is_disabled is False- summaries_data = from_data_uri(summaries.data)[1].decode("utf-8")- # Result is csv or json- assert summaries_data in ["a\n2\n12\n", '[{"a":2},{"a":12}]']# We don't have column summaries for non-dataframe dataassert summaries.stats["a"].min is Noneassert summaries.stats["a"].max is None@@ -923,11 +914,9 @@ def test_get_column_summaries_after_search() -> None:)def test_get_column_summaries_after_search_df(df: Any) -> None:table = ui.table(df)- summaries = table._get_column_summaries(- ColumnSummariesArgs(precompute=False)- )+ summaries = table._get_column_summaries(ColumnSummariesArgs())+ assert summaries.show_charts is Trueassert summaries.is_disabled is False- assert isinstance(summaries.data, str)# Different dataframe types return different formatsFORMATS = ["data:text/plain;base64,", # arrow format for polars@@ -935,7 +924,6 @@ def test_get_column_summaries_after_search_df(df: Any) -> None:"data:text/csv;base64,",]- assert any(summaries.data.startswith(fmt) for fmt in FORMATS)assert summaries.stats["a"].min == 0assert summaries.stats["a"].max == 19@@ -947,12 +935,9 @@ def test_get_column_summaries_after_search_df(df: Any) -> None:page_number=0,))- summaries = table._get_column_summaries(- ColumnSummariesArgs(precompute=False)- )+ summaries = table._get_column_summaries(ColumnSummariesArgs())+ assert summaries.show_charts is Trueassert summaries.is_disabled is False- assert isinstance(summaries.data, str)- assert any(summaries.data.startswith(fmt) for fmt in FORMATS)# We don't have column summaries for non-dataframe dataassert summaries.stats["a"].min == 2assert summaries.stats["a"].max == 12@@ -964,40 +949,34 @@ def test_show_column_summaries_modes():# Test stats-only modetable_stats = ui.table(data, show_column_summaries="stats")- summaries_stats = table_stats._get_column_summaries(- ColumnSummariesArgs(precompute=False)- )+ summaries_stats = table_stats._get_column_summaries(ColumnSummariesArgs())+ assert summaries_stats.show_charts is Falseassert summaries_stats.is_disabled is False- assert summaries_stats.data is Noneassert summaries_stats.bin_values == {}assert summaries_stats.value_counts == {}assert len(summaries_stats.stats) > 0# Test chart-only modetable_chart = ui.table(data, show_column_summaries="chart")- summaries_chart = table_chart._get_column_summaries(- ColumnSummariesArgs(precompute=False)- )+ summaries_chart = table_chart._get_column_summaries(ColumnSummariesArgs())+ assert summaries_chart.show_charts is Trueassert summaries_chart.is_disabled is False- assert summaries_chart.data is not Noneassert len(summaries_chart.stats) == 0# Test default mode (both stats and chart)table_both = ui.table(data, show_column_summaries=True)- summaries_both = table_both._get_column_summaries(- ColumnSummariesArgs(precompute=False)- )+ summaries_both = table_both._get_column_summaries(ColumnSummariesArgs())+ assert summaries_both.show_charts is Trueassert summaries_both.is_disabled is False- assert summaries_both.data is not Noneassert len(summaries_both.stats) > 0# Test disabled modetable_disabled = ui.table(data, show_column_summaries=False)summaries_disabled = table_disabled._get_column_summaries(- ColumnSummariesArgs(precompute=False)+ ColumnSummariesArgs())assert summaries_disabled.is_disabled is False- assert summaries_disabled.data is None+ assert summaries_disabled.show_charts is Falseassert summaries_disabled.bin_values == {}assert summaries_disabled.value_counts == {}assert len(summaries_disabled.stats) == 0@@ -1005,10 +984,10 @@ def test_show_column_summaries_modes():# Test Default behaviortable_default = ui.table(data)summaries_default = table_default._get_column_summaries(- ColumnSummariesArgs(precompute=False)+ ColumnSummariesArgs())+ assert summaries_default.show_charts is Trueassert summaries_default.is_disabled is False- assert summaries_default.data is not Noneassert len(summaries_default.stats) > 0assert table_default._component_args["show-column-summaries"] is True@@ -1020,9 +999,7 @@ class TestTableBinValues:)def test_bin_values_all_nulls(self, df: Any) -> None:table = ui.table(df)- summaries = table._get_column_summaries(- ColumnSummariesArgs(precompute=True)- )+ summaries = table._get_column_summaries(ColumnSummariesArgs())# Returns empty listassert summaries.bin_values == {"a": []}@@ -1067,6 +1044,41 @@ class TestTableGetValueCounts:ValueCount(value="4", count=1),]+ @pytest.mark.skipif(+ not DependencyManager.pandas.has(), reason="Pandas not installed"+ )+ def test_rows_string_value_counts_limit(self) -> None:+ import pandas as pd++ data = pd.DataFrame(+ {"a": [str(i) for i in range(CHART_MAX_ROWS_STRING_VALUE_COUNTS)]}+ )+ table = ui.table(data)+ summaries = table._get_column_summaries(ColumnSummariesArgs())+ assert summaries.value_counts == {+ "a": [+ ValueCount(+ value="unique values",+ count=CHART_MAX_ROWS_STRING_VALUE_COUNTS,+ )+ ]+ }+ assert summaries.data is None++ # If >20k rows, we should not get value_counts+ data = pd.DataFrame(+ {+ "a": [+ str(i)+ for i in range(CHART_MAX_ROWS_STRING_VALUE_COUNTS + 1)+ ]+ }+ )+ table = ui.table(data)+ summaries = table._get_column_summaries(ColumnSummariesArgs())+ assert summaries.data is None+ assert summaries.value_counts == {}+def test_with_smaller_limit(self, table: ui.table) -> None:value_counts = table._get_value_counts(column="repeat", size=2, total_rows=self.total_rows@@ -1177,10 +1189,40 @@ def test_show_column_summaries_disabled():summaries = table._get_column_summaries(EmptyArgs())assert summaries.is_disabled is False- assert summaries.data is Noneassert len(summaries.stats) == 0+ not DependencyManager.polars.has(), reason="Polars not installed"+)+def test_column_summaries_fallback(monkeypatch):+ import polars as pl++ data = pl.DataFrame(+ {+ "a": [1, 2, 3] * 200,+ "b": [4, 5, 6] * 200,+ "c": [7, 8, 9] * 200,+ }+ )+ table = ui.table(data)++ def always_fail_get_bin_values(*_args: Any, **_kwargs: Any) -> None:+ raise RuntimeError("Intentional bin failure")++ monkeypatch.setattr(+ table._manager, "get_bin_values", always_fail_get_bin_values+ )++ summaries = table._get_column_summaries(ColumnSummariesArgs())+ assert summaries.is_disabled is False+ assert summaries.bin_values == {}+ assert summaries.value_counts == {}+ assert summaries.show_charts is True+ # Should have chart data+ assert summaries.data is not None++@pytest.mark.parametrize("df",create_dataframes(