A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
Feat add pivot transform to mo.UI.dataframe #7345 (#7584)
Heizer Balazsea749af2e5fc
diff --git a/docs/api/inputs/dataframe.md b/docs/api/inputs/dataframe.mdindex 902a055688..9d3b4aa67f 100644--- a/docs/api/inputs/dataframe.md+++ b/docs/api/inputs/dataframe.md@@ -20,6 +20,8 @@ Supported transforms are:- Shuffle Rows- Explode Columns- Expand Dict+- Unique+- Pivot!!! info "Request a Transform"diff --git a/frontend/src/plugins/impl/data-frames/forms/__tests__/__snapshots__/form.test.tsx.snap b/frontend/src/plugins/impl/data-frames/forms/__tests__/__snapshots__/form.test.tsx.snapindex 77c6e22762..2d12424868 100644--- a/frontend/src/plugins/impl/data-frames/forms/__tests__/__snapshots__/form.test.tsx.snap+++ b/frontend/src/plugins/impl/data-frames/forms/__tests__/__snapshots__/form.test.tsx.snap@@ -20,7 +20,7 @@ exports[`renderZodSchema > should render a form aggregate 1`] = `class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"for="_r_26_-form-item">- Group by columns+ Columns</label><divaria-describedby="_r_26_-form-item-description"@@ -39,7 +39,7 @@ exports[`renderZodSchema > should render a form aggregate 1`] = `<spanclass="truncate flex-1 min-w-0">- Select group by columns+ Select columns</span><svgaria-hidden="true"@@ -392,7 +392,7 @@ exports[`renderZodSchema > should render a form explode_columns 1`] = `class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"for="_r_2o_-form-item">- Group by columns+ Columns</label><divaria-describedby="_r_2o_-form-item-description"@@ -411,7 +411,7 @@ exports[`renderZodSchema > should render a form explode_columns 1`] = `<spanclass="truncate flex-1 min-w-0">- Select group by columns+ Select columns</span><svgaria-hidden="true"@@ -771,6 +771,233 @@ exports[`renderZodSchema > should render a form group_by 1`] = `</DocumentFragment>`;+exports[`renderZodSchema > should render a form pivot 1`] = `+<DocumentFragment>+ <div+ class="flex flex-col gap-6"+ >+ <input+ name="type"+ type="hidden"+ value="pivot"+ />+ <div+ class="flex flex-row align-start"+ >+ <div+ class="flex flex-col gap-1"+ >+ <label+ class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"+ for="_r_3a_-form-item"+ >+ Columns+ </label>+ <div+ aria-describedby="_r_3a_-form-item-description"+ aria-invalid="false"+ class="relative"+ id="_r_3a_-form-item"+ >+ <div+ aria-controls="radix-_r_3b_"+ aria-expanded="false"+ aria-haspopup="dialog"+ class="flex h-6 w-fit mb-1 shadow-xs-solid items-center justify-between rounded-sm border border-input bg-transparent px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground hover:shadow-sm-solid focus:outline-hidden focus:ring-1 focus:ring-ring focus:border-primary focus:shadow-md-solid disabled:cursor-not-allowed disabled:opacity-50 min-w-[180px]"+ data-state="closed"+ type="button"+ >+ <span+ class="truncate flex-1 min-w-0"+ >+ Select columns+ </span>+ <svg+ aria-hidden="true"+ class="lucide lucide-chevron-down ml-3 w-4 h-4 opacity-50 shrink-0"+ fill="none"+ height="24"+ stroke="currentColor"+ stroke-linecap="round"+ stroke-linejoin="round"+ stroke-width="2"+ viewBox="0 0 24 24"+ width="24"+ xmlns="http://www.w3.org/2000/svg"+ >+ <path+ d="m6 9 6 6 6-6"+ />+ </svg>+ </div>+ <div+ class="flex flex-col gap-1 items-start"+ />+ </div>+ </div>+ </div>+ <div+ class="flex flex-row align-start"+ >+ <div+ class="flex flex-col gap-1"+ >+ <label+ class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"+ for="_r_3c_-form-item"+ >+ Rows+ </label>+ <div+ aria-describedby="_r_3c_-form-item-description"+ aria-invalid="false"+ class="relative"+ id="_r_3c_-form-item"+ >+ <div+ aria-controls="radix-_r_3d_"+ aria-expanded="false"+ aria-haspopup="dialog"+ class="flex h-6 w-fit mb-1 shadow-xs-solid items-center justify-between rounded-sm border border-input bg-transparent px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground hover:shadow-sm-solid focus:outline-hidden focus:ring-1 focus:ring-ring focus:border-primary focus:shadow-md-solid disabled:cursor-not-allowed disabled:opacity-50 min-w-[180px]"+ data-state="closed"+ type="button"+ >+ <span+ class="truncate flex-1 min-w-0"+ >+ Select rows+ </span>+ <svg+ aria-hidden="true"+ class="lucide lucide-chevron-down ml-3 w-4 h-4 opacity-50 shrink-0"+ fill="none"+ height="24"+ stroke="currentColor"+ stroke-linecap="round"+ stroke-linejoin="round"+ stroke-width="2"+ viewBox="0 0 24 24"+ width="24"+ xmlns="http://www.w3.org/2000/svg"+ >+ <path+ d="m6 9 6 6 6-6"+ />+ </svg>+ </div>+ <div+ class="flex flex-col gap-1 items-start"+ />+ </div>+ </div>+ </div>+ <div+ class="flex flex-row align-start"+ >+ <div+ class="flex flex-col gap-1"+ >+ <label+ class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"+ for="_r_3e_-form-item"+ >+ Values+ </label>+ <div+ aria-describedby="_r_3e_-form-item-description"+ aria-invalid="false"+ class="relative"+ id="_r_3e_-form-item"+ >+ <div+ aria-controls="radix-_r_3f_"+ aria-expanded="false"+ aria-haspopup="dialog"+ class="flex h-6 w-fit mb-1 shadow-xs-solid items-center justify-between rounded-sm border border-input bg-transparent px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground hover:shadow-sm-solid focus:outline-hidden focus:ring-1 focus:ring-ring focus:border-primary focus:shadow-md-solid disabled:cursor-not-allowed disabled:opacity-50 min-w-[180px]"+ data-state="closed"+ type="button"+ >+ <span+ class="truncate flex-1 min-w-0"+ >+ Select values+ </span>+ <svg+ aria-hidden="true"+ class="lucide lucide-chevron-down ml-3 w-4 h-4 opacity-50 shrink-0"+ fill="none"+ height="24"+ stroke="currentColor"+ stroke-linecap="round"+ stroke-linejoin="round"+ stroke-width="2"+ viewBox="0 0 24 24"+ width="24"+ xmlns="http://www.w3.org/2000/svg"+ >+ <path+ d="m6 9 6 6 6-6"+ />+ </svg>+ </div>+ <div+ class="flex flex-col gap-1 items-start"+ />+ </div>+ </div>+ </div>+ <div+ class="flex flex-row align-start"+ >+ <div+ class="flex flex-col gap-1"+ >+ <label+ class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 whitespace-pre"+ for="_r_3g_-form-item"+ >+ Aggregation+ </label>+ <button+ aria-autocomplete="none"+ aria-controls="radix-_r_3h_"+ aria-expanded="false"+ class="flex h-6 w-fit items-center justify-between rounded-sm bg-background px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground focus:outline-hidden focus:ring-1 focus:ring-ring disabled:cursor-not-allowed disabled:opacity-50 cursor-pointer shadow-xs-solid border border-input hover:shadow-sm-solid focus:border-primary focus:shadow-md-solid disabled:hover:shadow-xs-solid mb-0 min-w-[180px]"+ data-placeholder=""+ data-state="closed"+ dir="ltr"+ role="combobox"+ type="button"+ >+ <span+ style="pointer-events: none;"+ >+ --+ </span>+ <svg+ aria-hidden="true"+ class="lucide lucide-chevron-down h-4 w-4 opacity-50"+ fill="none"+ height="24"+ stroke="currentColor"+ stroke-linecap="round"+ stroke-linejoin="round"+ stroke-width="2"+ viewBox="0 0 24 24"+ width="24"+ xmlns="http://www.w3.org/2000/svg"+ >+ <path+ d="m6 9 6 6 6-6"+ />+ </svg>+ </button>+ </div>+ </div>+ </div>+</DocumentFragment>+`;+exports[`renderZodSchema > should render a form rename_column 1`] = `<DocumentFragment><div@@ -1064,7 +1291,7 @@ exports[`renderZodSchema > should render a form select_columns 1`] = `class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"for="_r_3_-form-item">- Group by columns+ Columns</label><divaria-describedby="_r_3_-form-item-description"@@ -1083,7 +1310,7 @@ exports[`renderZodSchema > should render a form select_columns 1`] = `<spanclass="truncate flex-1 min-w-0">- Select group by columns+ Select columns</span><svgaria-hidden="true"@@ -1347,7 +1574,7 @@ exports[`renderZodSchema > should render a form unique 1`] = `class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"for="_r_31_-form-item">- Group by columns+ Columns</label><divaria-describedby="_r_31_-form-item-description"@@ -1366,7 +1593,7 @@ exports[`renderZodSchema > should render a form unique 1`] = `<spanclass="truncate flex-1 min-w-0">- Select group by columns+ Select columns</span><svgaria-hidden="true"@@ -1451,7 +1678,7 @@ exports[`renders custom forms column_id 1`] = `><labelclass="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"- for="_r_3a_-form-item"+ for="_r_3p_-form-item">Column</label>@@ -1460,7 +1687,7 @@ exports[`renders custom forms column_id 1`] = `><buttonaria-autocomplete="none"- aria-controls="radix-_r_3b_"+ aria-controls="radix-_r_3q_"aria-expanded="false"class="flex h-6 w-fit items-center justify-between rounded-sm bg-background px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground focus:outline-hidden focus:ring-1 focus:ring-ring disabled:cursor-not-allowed disabled:opacity-50 cursor-pointer shadow-xs-solid border border-input hover:shadow-sm-solid focus:border-primary focus:shadow-md-solid disabled:hover:shadow-xs-solid mb-0 min-w-[180px]"data-placeholder=""@@ -1504,18 +1731,18 @@ exports[`renders custom forms column_id_array 1`] = `><labelclass="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"- for="_r_3h_-form-item"+ for="_r_40_-form-item">- Group by columns+ Columns</label><div- aria-describedby="_r_3h_-form-item-description"+ aria-describedby="_r_40_-form-item-description"aria-invalid="false"class="relative"- id="_r_3h_-form-item"+ id="_r_40_-form-item"><div- aria-controls="radix-_r_3i_"+ aria-controls="radix-_r_41_"aria-expanded="false"aria-haspopup="dialog"class="flex h-6 w-fit mb-1 shadow-xs-solid items-center justify-between rounded-sm border border-input bg-transparent px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground hover:shadow-sm-solid focus:outline-hidden focus:ring-1 focus:ring-ring focus:border-primary focus:shadow-md-solid disabled:cursor-not-allowed disabled:opacity-50 min-w-[180px]"@@ -1525,7 +1752,7 @@ exports[`renders custom forms column_id_array 1`] = `<spanclass="truncate flex-1 min-w-0">- Select group by columns+ Select columns</span><svgaria-hidden="true"@@ -1559,13 +1786,13 @@ exports[`renders custom forms column_id_dot_array 1`] = `class="flex flex-col gap-1"><div- aria-describedby="_r_3j_-form-item-description"+ aria-describedby="_r_42_-form-item-description"aria-invalid="false"class="relative"- id="_r_3j_-form-item"+ id="_r_42_-form-item"><div- aria-controls="radix-_r_3k_"+ aria-controls="radix-_r_43_"aria-expanded="false"aria-haspopup="dialog"class="flex h-6 w-fit mb-1 shadow-xs-solid items-center justify-between rounded-sm border border-input bg-transparent px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground hover:shadow-sm-solid focus:outline-hidden focus:ring-1 focus:ring-ring focus:border-primary focus:shadow-md-solid disabled:cursor-not-allowed disabled:opacity-50 min-w-[180px]"@@ -1610,7 +1837,7 @@ exports[`renders custom forms column_id_optional 1`] = `><labelclass="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"- for="_r_3l_-form-item"+ for="_r_44_-form-item">Column</label>@@ -1619,7 +1846,7 @@ exports[`renders custom forms column_id_optional 1`] = `><buttonaria-autocomplete="none"- aria-controls="radix-_r_3m_"+ aria-controls="radix-_r_45_"aria-expanded="false"class="flex h-6 w-fit items-center justify-between rounded-sm bg-background px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground focus:outline-hidden focus:ring-1 focus:ring-ring disabled:cursor-not-allowed disabled:opacity-50 cursor-pointer shadow-xs-solid border border-input hover:shadow-sm-solid focus:border-primary focus:shadow-md-solid disabled:hover:shadow-xs-solid mb-0 min-w-[180px]"data-placeholder=""diff --git a/frontend/src/plugins/impl/data-frames/panel.tsx b/frontend/src/plugins/impl/data-frames/panel.tsxindex d27101d4a1..482ed89d93 100644--- a/frontend/src/plugins/impl/data-frames/panel.tsx+++ b/frontend/src/plugins/impl/data-frames/panel.tsx@@ -17,6 +17,7 @@ import {PlusIcon,ShuffleIcon,SquareMousePointerIcon,+ Table2Icon,Trash2Icon,} from "lucide-react";import React, {@@ -394,4 +395,5 @@ const ICONS: Record<TransformType["type"], React.FC<{ className?: string }>> = {explode_columns: BracketsIcon,expand_dict: FileJsonIcon,unique: CopySlashIcon,+ pivot: Table2Icon,};diff --git a/frontend/src/plugins/impl/data-frames/schema.ts b/frontend/src/plugins/impl/data-frames/schema.tsindex 98c38c80fd..f9bd569e89 100644--- a/frontend/src/plugins/impl/data-frames/schema.ts+++ b/frontend/src/plugins/impl/data-frames/schema.ts@@ -29,7 +29,7 @@ export const column_id_array = z.array(column_id.describe(FieldOptions.of({ special: "column_id" }))).min(1, "At least one column is required").default([])- .describe(FieldOptions.of({ label: "Group by columns", minLength: 1 }));+ .describe(FieldOptions.of({ label: "Columns", minLength: 1 }));const ColumnConversionTransformSchema = z.object({@@ -107,7 +107,10 @@ const FilterRowsTransformSchema = z.object({const GroupByTransformSchema = z.object({type: z.literal("group_by"),- column_ids: column_id_array,+ column_ids: z+ .array(column_id.describe(FieldOptions.of({ special: "column_id" })))+ .default([])+ .describe(FieldOptions.of({ label: "Group by columns", minLength: 1 })),aggregation_column_ids: z.array(column_id.describe(FieldOptions.of({ special: "column_id" }))).default([])@@ -193,6 +196,25 @@ const UniqueTransformSchema = z}).describe(FieldOptions.of({ direction: "row" }));+const PivotTransformSchema = z+ .object({+ type: z.literal("pivot"),+ column_ids: column_id_array,+ index_column_ids: z+ .array(column_id.describe(FieldOptions.of({ special: "column_id" })))+ .default([])+ .describe(FieldOptions.of({ label: "Rows" })),+ value_column_ids: z+ .array(column_id.describe(FieldOptions.of({ special: "column_id" })))+ .default([])+ .describe(FieldOptions.of({ label: "Values", minLength: 1 })),+ aggregation: z+ .enum(AGGREGATION_FNS)+ .default("sum")+ .describe(FieldOptions.of({ label: "Aggregation" })),+ })+ .describe(FieldOptions.of({}));+export const TransformTypeSchema = z.union([FilterRowsTransformSchema,SelectColumnsTransformSchema,@@ -206,6 +228,7 @@ export const TransformTypeSchema = z.union([ExplodeColumnsTransformSchema,ExpandDictTransformSchema,UniqueTransformSchema,+ PivotTransformSchema,]);export type TransformType = z.infer<typeof TransformTypeSchema>;diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/apply.py b/marimo/_plugins/ui/_impl/dataframes/transforms/apply.pyindex 113a9ed823..8cedb0318d 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/apply.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/apply.py@@ -51,6 +51,8 @@ def _handle(df: T, handler: TransformHandler[T], transform: Transform) -> T:return handler.handle_expand_dict(df, transform)if transform.type is TransformType.UNIQUE:return handler.handle_unique(df, transform)+ if transform.type is TransformType.PIVOT:+ return handler.handle_pivot(df, transform)assert_never(transform.type)diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.pyindex de6d4845ea..6e5ef748bb 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py@@ -2,6 +2,7 @@from __future__ import annotationsimport datetime+from functools import reducefrom typing import TYPE_CHECKING, Any, Callableimport narwhals.stable.v2 as nw@@ -21,6 +22,7 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (ExplodeColumnsTransform,FilterRowsTransform,GroupByTransform,+ PivotTransform,RenameColumnTransform,SampleRowsTransform,SelectColumnsTransform,@@ -370,6 +372,89 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):)assert_never(keep)+ @staticmethod+ def handle_pivot(df: DataFrame, transform: PivotTransform) -> DataFrame:+ # Since ibis does not have a native pivot, and pivot is not supported for LazyFrame+ # we implement it manually+ # pivot results are also highly inconsistent across backends, so we standardize the output here++ if not transform.index_column_ids and not transform.value_column_ids:+ raise nw.exceptions.InvalidOperationError(+ "Pivot transform requires at least one index column and or value column."+ )++ columns = df.collect_schema().names()+ if not transform.index_column_ids:+ index_columns = list(+ filter(+ lambda col: col not in transform.column_ids+ and col not in transform.value_column_ids,+ columns,+ )+ )+ else:+ index_columns = transform.index_column_ids++ if not transform.value_column_ids:+ value_columns = list(+ filter(+ lambda col: col not in transform.column_ids+ and col not in transform.index_column_ids,+ columns,+ )+ )+ else:+ value_columns = transform.value_column_ids++ raw_pivot_columns = (+ df.select(*transform.column_ids)+ .unique()+ .sort(by=transform.column_ids)+ .collect()+ .rows()+ )++ dfs = []+ for raw_pivot_column in raw_pivot_columns:+ aggs = []+ mask = reduce(+ lambda x, y: x & y,+ [+ nw.col(on_col) == on_val+ for on_col, on_val in zip(+ transform.column_ids, raw_pivot_column+ )+ ],+ )+ for value_column in value_columns:+ expr = nw.col(value_column).alias(+ f"{value_column}_{'_'.join(map(str, raw_pivot_column))}_{transform.aggregation}"+ )+ if transform.aggregation == "count":+ aggs.append(expr.len())+ elif transform.aggregation == "sum":+ aggs.append(expr.sum())+ elif transform.aggregation == "mean":+ aggs.append(expr.mean())+ elif transform.aggregation == "median":+ aggs.append(expr.median())+ elif transform.aggregation == "min":+ aggs.append(expr.min())+ elif transform.aggregation == "max":+ aggs.append(expr.max())+ else:+ raise ValueError(+ f"Unsupported aggregation function: {transform.aggregation}"+ )+ dfs.append(df.filter(mask).group_by(*index_columns).agg(*aggs))++ result = df.select(*index_columns).unique()+ for df_ in dfs:+ result = result.join(df_, on=index_columns, how="left")+ if transform.aggregation in {"count", "sum"}:+ result = result.select(nw.all().fill_null(0))+ return result.sort(by=index_columns)+@staticmethoddef as_python_code(df: DataFrame,diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.pyindex e4768735e8..17e9497889 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py@@ -217,6 +217,52 @@ def python_print_pandas(column_ids = transform.column_idsreturn f"{df_name}.drop_duplicates({_list_of_strings(column_ids)}, keep={_as_literal(transform.keep)})"+ elif transform.type == TransformType.PIVOT:+ if not transform.index_column_ids:+ index_columns = _list_of_strings(+ list(+ filter(+ lambda col: col not in transform.column_ids+ and col not in transform.value_column_ids,+ all_columns,+ )+ )+ )+ else:+ index_columns = _list_of_strings(transform.index_column_ids)++ if not transform.value_column_ids:+ value_columns = _list_of_strings(+ list(+ filter(+ lambda col: col not in transform.column_ids+ and col not in transform.index_column_ids,+ all_columns,+ )+ )+ )+ else:+ value_columns = _list_of_strings(transform.value_column_ids)+ column_ids = transform.column_ids+ agg_func = transform.aggregation++ args = _args_list(+ f"index={index_columns}",+ f"columns={_list_of_strings(column_ids)}",+ f"values={value_columns}",+ f"aggfunc={_as_literal(agg_func)}",+ "sort=False",+ )+ pivot_code = f"{df_name}.pivot_table({args}).sort_index(axis=0)"+ flatten_columns_code = (+ f"{df_name}.columns = ["+ f"f\"{{'_'.join(map(str, col)).strip()}}_{agg_func}\" "+ f'if isinstance(col, tuple) else f"{{col}}_{agg_func}" '+ f"for col in {df_name}.columns]"+ )+ reset_index_code = f"{df_name} = {df_name}.reset_index()"+ return f"{pivot_code}\n{flatten_columns_code}\n{reset_index_code}"+assert_never(transform.type)@@ -378,6 +424,56 @@ def python_print_polars(column_ids = transform.column_idsreturn f"{df_name}.unique(subset={_list_of_strings(column_ids)}, keep={_as_literal(transform.keep)})" # noqa: E501+ elif transform.type == TransformType.PIVOT:+ if not transform.index_column_ids:+ index_column_ids = _list_of_strings(+ list(+ filter(+ lambda col: col not in transform.column_ids+ and col not in transform.value_column_ids,+ all_columns,+ )+ )+ )+ else:+ index_column_ids = _list_of_strings(transform.index_column_ids)++ if not transform.value_column_ids:+ value_column_ids = _list_of_strings(+ list(+ filter(+ lambda col: col not in transform.column_ids+ and col not in transform.index_column_ids,+ all_columns,+ )+ )+ )+ else:+ value_column_ids = _list_of_strings(transform.value_column_ids)++ args = _args_list(+ f"on={_list_of_strings(transform.column_ids)}",+ f"index={index_column_ids}",+ f"values={value_column_ids}",+ f"aggregate_function={_as_literal(transform.aggregation) if transform.aggregation != 'count' else _as_literal('len')}",+ )+ fill_null_code = (+ ".select(pl.all().fill_null(0))"+ if transform.aggregation in ["count", "sum"]+ else ""+ ) # noqa: E501+ pivot_code = f"{df_name}{fill_null_code}.pivot({args}).sort(by={index_column_ids})" # noqa: E501+ lambda_code = (+ f'lambda col,replacements=replacements: f"{transform.value_column_ids[0]}_{{col.translate(replacements)}}_{transform.aggregation}"' # noqa: E501+ if len(transform.value_column_ids) == 1+ else f"lambda col, replacements=replacements: f'{{col.translate(replacements)}}_{transform.aggregation}'"+ ) + f" if col not in {index_column_ids} else col"+ rename_code = (+ 'replacements = str.maketrans({"{": "", "}": "", \'"\': "", ",": "_"})\n'+ f"{df_name} = {df_name}.rename({lambda_code})"+ )+ return f"{pivot_code}\n{rename_code}"+assert_never(transform.type)@@ -508,6 +604,48 @@ def python_print_ibis(column_ids = transform.column_idsreturn f"{df_name}.distinct(on={_list_of_strings(column_ids)}, keep={_as_literal(transform.keep)})" # noqa: E501+ elif transform.type == TransformType.PIVOT:+ if not transform.index_column_ids:+ index_column_ids = _list_of_strings(+ list(+ filter(+ lambda col: col not in transform.column_ids+ and col not in transform.value_column_ids,+ all_columns,+ )+ )+ )+ else:+ index_column_ids = _list_of_strings(transform.index_column_ids)++ if not transform.value_column_ids:+ value_column_ids = _list_of_strings(+ list(+ filter(+ lambda col: col not in transform.column_ids+ and col not in transform.index_column_ids,+ all_columns,+ )+ )+ )+ else:+ value_column_ids = _list_of_strings(transform.value_column_ids)++ column_ids = transform.column_ids+ agg_func = transform.aggregation++ pivot_code = (+ f"{df_name}.pivot_wider("+ f"names_from={_list_of_strings(column_ids)}, "+ f"id_cols={_list_of_strings(index_column_ids)}, "+ f"values_from={_list_of_strings(value_column_ids)}, "+ f"names_prefix={_as_literal(value_column_ids[0]) if len(value_column_ids) == 1 else _as_literal('')}, "+ f"values_agg={_as_literal(agg_func)})"+ )++ rename_code = f'{df_name} = {df_name}.rename(**{{f"{{col}}_{agg_func}": col for col in {df_name}.columns if col not in {_list_of_strings(index_column_ids)}}})' # noqa: E501+ return f"{pivot_code}\n{rename_code}"+assert_never(transform.type)diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/types.py b/marimo/_plugins/ui/_impl/dataframes/transforms/types.pyindex 754e72c506..e1bb565589 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/types.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/types.py@@ -66,6 +66,7 @@ class TransformType(Enum):EXPLODE_COLUMNS = "explode_columns"EXPAND_DICT = "expand_dict"UNIQUE = "unique"+ PIVOT = "pivot"@dataclass(frozen=True)@@ -176,10 +177,20 @@ class UniqueTransform:keep: UniqueKeep+@dataclass+class PivotTransform:+ type: Literal[TransformType.PIVOT]+ column_ids: ColumnIds+ index_column_ids: ColumnIds+ value_column_ids: ColumnIds+ aggregation: Aggregation++Transform = Union[AggregateTransform,ColumnConversionTransform,FilterRowsTransform,+ PivotTransform,GroupByTransform,RenameColumnTransform,SelectColumnsTransform,@@ -263,6 +274,11 @@ class TransformHandler(abc.ABC, Generic[T]):def handle_unique(df: T, transform: UniqueTransform) -> T:raise NotImplementedError+ @staticmethod+ @abc.abstractmethod+ def handle_pivot(df: T, transform: PivotTransform) -> T:+ raise NotImplementedError+@staticmethoddef as_python_code(df: T, df_name: str, columns: list[str], transforms: list[Transform]diff --git a/marimo/_smoke_tests/issues/7345-dataframe-pivot-transform.py b/marimo/_smoke_tests/issues/7345-dataframe-pivot-transform.pynew file mode 100644index 0000000000..98af96c558--- /dev/null+++ b/marimo/_smoke_tests/issues/7345-dataframe-pivot-transform.py@@ -0,0 +1,108 @@+import marimo++__generated_with = "0.18.4"+app = marimo.App(width="columns")++[email protected](column=0)+def _():+ import marimo as mo+ import polars as pl+ import pandas as pd+ import ibis as ib++ from vega_datasets import data+ return data, ib, mo, pd, pl+++def _(data, ib, pd, pl):+ cars = data.cars()+ df_pandas = pd.DataFrame(cars)+ df_polars = pl.DataFrame(cars)+ df_ibis = ib.memtable(cars)+ return df_ibis, df_pandas, df_polars++[email protected](column=1)+def _(df_pandas, mo):+ mo.ui.dataframe(df_pandas)+ return+++def _(df_polars, mo):+ mo.ui.dataframe(df_polars)+ return+++def _(df_ibis, mo):+ mo.ui.dataframe(df_ibis)+ return++[email protected](column=2)+def _(df_pandas):+ df_pandas_next = df_pandas+ df_pandas_next = df_pandas_next.pivot_table(+ index=["Year"],+ columns=["Origin"],+ values=["Acceleration"],+ aggfunc="mean",+ sort=False,+ fill_value=None,+ ).sort_index(axis=0)+ df_pandas_next.columns = [+ f"{'_'.join(map(str, col)).strip()}_mean"+ if isinstance(col, tuple)+ else f"{col}_mean"+ for col in df_pandas_next.columns+ ]+ df_pandas_next = df_pandas_next.reset_index()+ df_pandas_next+ return+++def _(df_polars):+ df_polars_next = df_polars+ df_polars_next = df_polars_next.pivot(+ on=["Origin"],+ index=["Year"],+ values=["Acceleration"],+ aggregate_function="mean",+ ).sort(["Year"])+ replacements = str.maketrans({"{": "", "}": "", '"': "", ",": "_"})+ df_polars_next = df_polars_next.rename(+ lambda col: f"Acceleration_{col.translate(replacements)}_mean"+ if col not in ["Year"]+ else col+ )+ df_polars_next+ return+++def _(df_ibis):+ df_ibis_next = df_ibis+ df_ibis_next = df_ibis_next.pivot_wider(+ names_from=["Origin"],+ id_cols=["Year"],+ values_from=["Acceleration"],+ names_prefix="Acceleration",+ values_agg="mean",+ )+ df_ibis_next = df_ibis_next.rename(+ **{+ f"{col}_mean": col+ for col in df_ibis_next.columns+ if col not in ["Year"]+ }+ )+ df_ibis_next.to_polars()+ return+++if __name__ == "__main__":+ app.run()diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.pyindex 32dabb4757..b9090957e7 100644--- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py+++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py@@ -23,6 +23,7 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (ExplodeColumnsTransform,FilterRowsTransform,GroupByTransform,+ PivotTransform,RenameColumnTransform,SampleRowsTransform,SelectColumnsTransform,@@ -1184,6 +1185,208 @@ class TestTransformHandler:)assert type(result) is type(df)+ @staticmethod+ @pytest.mark.parametrize(+ ("df", "expected", "transform"),+ [+ *zip(+ create_test_dataframes(+ {+ "A": [+ "foo",+ "foo",+ "foo",+ "foo",+ "foo",+ "bar",+ "bar",+ "bar",+ "bar",+ ],+ "B": [+ "one",+ "one",+ "one",+ "two",+ "two",+ "one",+ "one",+ "two",+ "two",+ ],+ "C": [+ "small",+ "large",+ "large",+ "small",+ "small",+ "large",+ "small",+ "small",+ "large",+ ],+ "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],+ }+ ),+ create_test_dataframes(+ {+ "B": ["one", "two"],+ "D_foo_sum": [5, 6],+ "D_bar_sum": [9, 13],+ }+ ),+ [+ PivotTransform(+ type=TransformType.PIVOT,+ column_ids=["A"],+ index_column_ids=["B"],+ value_column_ids=["D"],+ aggregation="sum",+ )+ ],+ ),+ *zip(+ create_test_dataframes(+ {+ "A": [+ "foo",+ "foo",+ "foo",+ "foo",+ "foo",+ "bar",+ "bar",+ "bar",+ "bar",+ ],+ "B": [+ "one",+ "one",+ "one",+ "two",+ "two",+ "one",+ "one",+ "two",+ "two",+ ],+ "C": [+ "small",+ "large",+ "large",+ "small",+ "small",+ "large",+ "small",+ "small",+ "large",+ ],+ "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],+ }+ ),+ create_test_dataframes(+ {+ "B": ["one", "one", "two", "two"],+ "C": ["large", "small", "large", "small"],+ "D_bar_sum": [4, 5, 7, 6],+ "D_foo_sum": [4, 1, None, 6],+ }+ ),+ [+ PivotTransform(+ type=TransformType.PIVOT,+ column_ids=["A"],+ index_column_ids=["B", "C"],+ value_column_ids=["D"],+ aggregation="sum",+ )+ ],+ ),+ *zip(+ create_test_dataframes(+ {+ "A": [+ "foo",+ "foo",+ "bar",+ "bar",+ ],+ "B": [+ "one",+ "two",+ "one",+ "two",+ ],+ "C": [+ "small",+ "large",+ "large",+ "small",+ ],+ "D": [1, 2, 3, 4],+ }+ ),+ create_test_dataframes(+ {+ "B": ["one", "one", "two", "two"],+ "C": ["large", "small", "large", "small"],+ "D_bar_sum": [3, None, None, 4],+ "D_foo_sum": [None, 1, 2, None],+ }+ ),+ [+ PivotTransform(+ type=TransformType.PIVOT,+ column_ids=["A"],+ index_column_ids=[],+ value_column_ids=["D"],+ aggregation="sum",+ )+ ],+ ),+ *zip(+ create_test_dataframes(+ {+ "A": [+ "foo",+ "foo",+ "bar",+ "bar",+ ],+ "B": [+ "one",+ "two",+ "one",+ "two",+ ],+ "D": [1, 2, 3, 4],+ }+ ),+ create_test_dataframes(+ {+ "B": ["one", "two"],+ "D_bar_sum": [3, 4],+ "D_foo_sum": [1, 2],+ }+ ),+ [+ PivotTransform(+ type=TransformType.PIVOT,+ column_ids=["A"],+ index_column_ids=["B"],+ value_column_ids=[],+ aggregation="sum",+ )+ ],+ ),+ ],+ )+ def test_handle_pivot(+ df: DataFrameType, expected: DataFrameType, transform: PivotTransform+ ) -> None:+ result = apply(df, transform)+ assert_frame_equal_with_nans(result, expected)+@staticmethod@pytest.mark.parametrize(("df", "expected", "expected2"),diff --git a/tests/_plugins/ui/_impl/dataframes/test_print_code.py b/tests/_plugins/ui/_impl/dataframes/test_print_code.pyindex addf78d18b..307dcd58e2 100644--- a/tests/_plugins/ui/_impl/dataframes/test_print_code.py+++ b/tests/_plugins/ui/_impl/dataframes/test_print_code.py@@ -30,6 +30,7 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (ExplodeColumnsTransform,FilterRowsTransform,GroupByTransform,+ PivotTransform,RenameColumnTransform,SampleRowsTransform,SelectColumnsTransform,@@ -75,10 +76,14 @@ def create_transform_strategy(list_column_id: Optional[st.SearchStrategy[str | int] | None] = None,df_size: int = 3,) -> st.SearchStrategy[Transform]:- column_ids = st.lists(column_id, min_size=1)+ column_ids = st.lists(column_id, min_size=1, unique=True)aggregation_column_ids = st.one_of(st.just([]),- st.lists(column_id, min_size=1),+ st.lists(column_id, min_size=1, unique=True),+ )+ index_column_ids = st.one_of(+ st.just([]),+ st.lists(column_id, min_size=1, unique=True),)if string_column_id is None:@@ -220,6 +225,15 @@ def create_transform_strategy(column_id=column_id,)+ pivot_transform_strategy = st.builds(+ PivotTransform,+ type=st.just(TransformType.PIVOT),+ column_ids=column_ids,+ index_column_ids=index_column_ids,+ aggregation=aggregation,+ value_column_ids=aggregation_column_ids,+ )+# Combine all transform strategiestransform_strategy = st.one_of(column_conversion_transform_strategy,@@ -233,6 +247,7 @@ def create_transform_strategy(sample_rows_transform_strategy,explode_columns_transform_strategy,expand_dict_transform_strategy,+ pivot_transform_strategy,)return transform_strategy@@ -366,6 +381,42 @@ def test_print_code_result_matches_actual_transform_pandas(# Ignore groupby meanif transform.type == TransformType.GROUP_BY:assume(transform.aggregation != "mean")+ assume("dict" not in transform.aggregation_column_ids)++ if transform.type == TransformType.PIVOT:+ assume(+ (+ set(transform.column_ids) & set(transform.index_column_ids)+ == set()+ )+ and (+ set(transform.column_ids) & set(transform.value_column_ids)+ == set()+ )+ and (+ set(transform.index_column_ids)+ & set(transform.value_column_ids)+ == set()+ )+ )+ assume(+ not any(+ column_id in {"lists", "dicts", "datetimes"}+ for column_id in transform.column_ids+ )+ )+ assume(+ not any(+ column_id in {"lists", "dicts", "datetimes"}+ for column_id in transform.index_column_ids+ )+ )+ assume(+ not any(+ column_id in {"lists", "dicts", "datetimes"}+ for column_id in transform.value_column_ids+ )+ )# Pandaspandas_code = python_print_transforms(@@ -423,6 +474,11 @@ def test_print_code_result_matches_actual_transform_pandas(if isinstance(real_result, pd.Series):real_result = real_result.to_frame()+ # For pivot transform the column order can be different, enforce column order by sorting.+ if transform.type == TransformType.PIVOT:+ code_result = code_result.loc[:, list(sorted(code_result.columns))]+ real_result = real_result.loc[:, list(sorted(real_result.columns))]+code_result = cast(pd.DataFrame, code_result).reset_index(drop=True)real_result = real_result.reset_index(drop=True)@@ -516,6 +572,40 @@ def test_print_code_result_matches_actual_transform_polars(# TODO: unimplementedif transform.type == TransformType.AGGREGATE:assume(False)+ if transform.type == TransformType.PIVOT:+ assume(+ (+ set(transform.column_ids) & set(transform.index_column_ids)+ == set()+ )+ and (+ set(transform.column_ids) & set(transform.value_column_ids)+ == set()+ )+ and (+ set(transform.index_column_ids)+ & set(transform.value_column_ids)+ == set()+ )+ )+ assume(+ not any(+ column_id in {"lists", "dicts", "datetimes"}+ for column_id in transform.column_ids+ )+ )+ assume(+ not any(+ column_id in {"lists", "dicts", "datetimes"}+ for column_id in transform.index_column_ids+ )+ )+ assume(+ not any(+ column_id in {"lists", "dicts", "datetimes"}+ for column_id in transform.value_column_ids+ )+ )# Polarspolars_code = python_print_transforms(@@ -575,6 +665,16 @@ def test_print_code_result_matches_actual_transform_polars(code_result = code_result.to_frame()if isinstance(real_result, pl.Series):real_result = real_result.to_frame()++ # For pivot transform the column order can be different, enforce column order by sorting.+ if transform.type == TransformType.PIVOT:+ code_result = code_result.select(+ *sorted(code_result.collect_schema().names())+ )+ real_result = real_result.select(+ *sorted(real_result.collect_schema().names())+ )+code_result = cast(pl.DataFrame, code_result)# Compare column namesassert code_result.columns == real_result.columns@@ -652,6 +752,41 @@ def test_print_code_result_matches_actual_transform_ibis(if transform.type == TransformType.COLUMN_CONVERSION:assume(transform.errors != "ignore")+ if transform.type == TransformType.PIVOT:+ assume(+ (+ set(transform.column_ids) & set(transform.index_column_ids)+ == set()+ )+ and (+ set(transform.column_ids) & set(transform.value_column_ids)+ == set()+ )+ and (+ set(transform.index_column_ids)+ & set(transform.value_column_ids)+ == set()+ )+ )+ assume(+ not any(+ column_id in {"lists", "dicts", "datetimes"}+ for column_id in transform.column_ids+ )+ )+ assume(+ not any(+ column_id in {"lists", "dicts", "datetimes"}+ for column_id in transform.index_column_ids+ )+ )+ assume(+ not any(+ column_id in {"lists", "dicts", "datetimes"}+ for column_id in transform.value_column_ids+ )+ )+try:nw_df = nw.from_native(my_df).lazy()result_nw = _apply_transforms(@@ -681,6 +816,11 @@ def test_print_code_result_matches_actual_transform_ibis(exec(ibis_code, {}, loc)code_result = loc.get("my_df_next")+ # For pivot transform the column order can be different, enforce column order by sorting.+ if transform.type == TransformType.PIVOT:+ code_result = code_result.select(*sorted(code_result.columns))+ real_result = real_result.select(*sorted(real_result.columns))+print("code_result", code_result)print("real_result", real_result)