Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

Feat add pivot transform to mo.UI.dataframe #7345 (#7584)

Heizer Balazsea749af2e5fc
diff --git a/docs/api/inputs/dataframe.md b/docs/api/inputs/dataframe.md
index 902a055688..9d3b4aa67f 100644
--- a/docs/api/inputs/dataframe.md
+++ b/docs/api/inputs/dataframe.md
@@ -20,6 +20,8 @@ Supported transforms are:
- Shuffle Rows
- Explode Columns
- Expand Dict
+- Unique
+- Pivot
!!! info "Request a Transform"
diff --git a/frontend/src/plugins/impl/data-frames/forms/__tests__/__snapshots__/form.test.tsx.snap b/frontend/src/plugins/impl/data-frames/forms/__tests__/__snapshots__/form.test.tsx.snap
index 77c6e22762..2d12424868 100644
--- a/frontend/src/plugins/impl/data-frames/forms/__tests__/__snapshots__/form.test.tsx.snap
+++ b/frontend/src/plugins/impl/data-frames/forms/__tests__/__snapshots__/form.test.tsx.snap
@@ -20,7 +20,7 @@ exports[`renderZodSchema > should render a form aggregate 1`] = `
class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
for="_r_26_-form-item"
>
- Group by columns
+ Columns
</label>
<div
aria-describedby="_r_26_-form-item-description"
@@ -39,7 +39,7 @@ exports[`renderZodSchema > should render a form aggregate 1`] = `
<span
class="truncate flex-1 min-w-0"
>
- Select group by columns
+ Select columns
</span>
<svg
aria-hidden="true"
@@ -392,7 +392,7 @@ exports[`renderZodSchema > should render a form explode_columns 1`] = `
class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
for="_r_2o_-form-item"
>
- Group by columns
+ Columns
</label>
<div
aria-describedby="_r_2o_-form-item-description"
@@ -411,7 +411,7 @@ exports[`renderZodSchema > should render a form explode_columns 1`] = `
<span
class="truncate flex-1 min-w-0"
>
- Select group by columns
+ Select columns
</span>
<svg
aria-hidden="true"
@@ -771,6 +771,233 @@ exports[`renderZodSchema > should render a form group_by 1`] = `
</DocumentFragment>
`;
+exports[`renderZodSchema > should render a form pivot 1`] = `
+<DocumentFragment>
+ <div
+ class="flex flex-col gap-6"
+ >
+ <input
+ name="type"
+ type="hidden"
+ value="pivot"
+ />
+ <div
+ class="flex flex-row align-start"
+ >
+ <div
+ class="flex flex-col gap-1"
+ >
+ <label
+ class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
+ for="_r_3a_-form-item"
+ >
+ Columns
+ </label>
+ <div
+ aria-describedby="_r_3a_-form-item-description"
+ aria-invalid="false"
+ class="relative"
+ id="_r_3a_-form-item"
+ >
+ <div
+ aria-controls="radix-_r_3b_"
+ aria-expanded="false"
+ aria-haspopup="dialog"
+ class="flex h-6 w-fit mb-1 shadow-xs-solid items-center justify-between rounded-sm border border-input bg-transparent px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground hover:shadow-sm-solid focus:outline-hidden focus:ring-1 focus:ring-ring focus:border-primary focus:shadow-md-solid disabled:cursor-not-allowed disabled:opacity-50 min-w-[180px]"
+ data-state="closed"
+ type="button"
+ >
+ <span
+ class="truncate flex-1 min-w-0"
+ >
+ Select columns
+ </span>
+ <svg
+ aria-hidden="true"
+ class="lucide lucide-chevron-down ml-3 w-4 h-4 opacity-50 shrink-0"
+ fill="none"
+ height="24"
+ stroke="currentColor"
+ stroke-linecap="round"
+ stroke-linejoin="round"
+ stroke-width="2"
+ viewBox="0 0 24 24"
+ width="24"
+ xmlns="http://www.w3.org/2000/svg"
+ >
+ <path
+ d="m6 9 6 6 6-6"
+ />
+ </svg>
+ </div>
+ <div
+ class="flex flex-col gap-1 items-start"
+ />
+ </div>
+ </div>
+ </div>
+ <div
+ class="flex flex-row align-start"
+ >
+ <div
+ class="flex flex-col gap-1"
+ >
+ <label
+ class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
+ for="_r_3c_-form-item"
+ >
+ Rows
+ </label>
+ <div
+ aria-describedby="_r_3c_-form-item-description"
+ aria-invalid="false"
+ class="relative"
+ id="_r_3c_-form-item"
+ >
+ <div
+ aria-controls="radix-_r_3d_"
+ aria-expanded="false"
+ aria-haspopup="dialog"
+ class="flex h-6 w-fit mb-1 shadow-xs-solid items-center justify-between rounded-sm border border-input bg-transparent px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground hover:shadow-sm-solid focus:outline-hidden focus:ring-1 focus:ring-ring focus:border-primary focus:shadow-md-solid disabled:cursor-not-allowed disabled:opacity-50 min-w-[180px]"
+ data-state="closed"
+ type="button"
+ >
+ <span
+ class="truncate flex-1 min-w-0"
+ >
+ Select rows
+ </span>
+ <svg
+ aria-hidden="true"
+ class="lucide lucide-chevron-down ml-3 w-4 h-4 opacity-50 shrink-0"
+ fill="none"
+ height="24"
+ stroke="currentColor"
+ stroke-linecap="round"
+ stroke-linejoin="round"
+ stroke-width="2"
+ viewBox="0 0 24 24"
+ width="24"
+ xmlns="http://www.w3.org/2000/svg"
+ >
+ <path
+ d="m6 9 6 6 6-6"
+ />
+ </svg>
+ </div>
+ <div
+ class="flex flex-col gap-1 items-start"
+ />
+ </div>
+ </div>
+ </div>
+ <div
+ class="flex flex-row align-start"
+ >
+ <div
+ class="flex flex-col gap-1"
+ >
+ <label
+ class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
+ for="_r_3e_-form-item"
+ >
+ Values
+ </label>
+ <div
+ aria-describedby="_r_3e_-form-item-description"
+ aria-invalid="false"
+ class="relative"
+ id="_r_3e_-form-item"
+ >
+ <div
+ aria-controls="radix-_r_3f_"
+ aria-expanded="false"
+ aria-haspopup="dialog"
+ class="flex h-6 w-fit mb-1 shadow-xs-solid items-center justify-between rounded-sm border border-input bg-transparent px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground hover:shadow-sm-solid focus:outline-hidden focus:ring-1 focus:ring-ring focus:border-primary focus:shadow-md-solid disabled:cursor-not-allowed disabled:opacity-50 min-w-[180px]"
+ data-state="closed"
+ type="button"
+ >
+ <span
+ class="truncate flex-1 min-w-0"
+ >
+ Select values
+ </span>
+ <svg
+ aria-hidden="true"
+ class="lucide lucide-chevron-down ml-3 w-4 h-4 opacity-50 shrink-0"
+ fill="none"
+ height="24"
+ stroke="currentColor"
+ stroke-linecap="round"
+ stroke-linejoin="round"
+ stroke-width="2"
+ viewBox="0 0 24 24"
+ width="24"
+ xmlns="http://www.w3.org/2000/svg"
+ >
+ <path
+ d="m6 9 6 6 6-6"
+ />
+ </svg>
+ </div>
+ <div
+ class="flex flex-col gap-1 items-start"
+ />
+ </div>
+ </div>
+ </div>
+ <div
+ class="flex flex-row align-start"
+ >
+ <div
+ class="flex flex-col gap-1"
+ >
+ <label
+ class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 whitespace-pre"
+ for="_r_3g_-form-item"
+ >
+ Aggregation
+ </label>
+ <button
+ aria-autocomplete="none"
+ aria-controls="radix-_r_3h_"
+ aria-expanded="false"
+ class="flex h-6 w-fit items-center justify-between rounded-sm bg-background px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground focus:outline-hidden focus:ring-1 focus:ring-ring disabled:cursor-not-allowed disabled:opacity-50 cursor-pointer shadow-xs-solid border border-input hover:shadow-sm-solid focus:border-primary focus:shadow-md-solid disabled:hover:shadow-xs-solid mb-0 min-w-[180px]"
+ data-placeholder=""
+ data-state="closed"
+ dir="ltr"
+ role="combobox"
+ type="button"
+ >
+ <span
+ style="pointer-events: none;"
+ >
+ --
+ </span>
+ <svg
+ aria-hidden="true"
+ class="lucide lucide-chevron-down h-4 w-4 opacity-50"
+ fill="none"
+ height="24"
+ stroke="currentColor"
+ stroke-linecap="round"
+ stroke-linejoin="round"
+ stroke-width="2"
+ viewBox="0 0 24 24"
+ width="24"
+ xmlns="http://www.w3.org/2000/svg"
+ >
+ <path
+ d="m6 9 6 6 6-6"
+ />
+ </svg>
+ </button>
+ </div>
+ </div>
+ </div>
+</DocumentFragment>
+`;
+
exports[`renderZodSchema > should render a form rename_column 1`] = `
<DocumentFragment>
<div
@@ -1064,7 +1291,7 @@ exports[`renderZodSchema > should render a form select_columns 1`] = `
class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
for="_r_3_-form-item"
>
- Group by columns
+ Columns
</label>
<div
aria-describedby="_r_3_-form-item-description"
@@ -1083,7 +1310,7 @@ exports[`renderZodSchema > should render a form select_columns 1`] = `
<span
class="truncate flex-1 min-w-0"
>
- Select group by columns
+ Select columns
</span>
<svg
aria-hidden="true"
@@ -1347,7 +1574,7 @@ exports[`renderZodSchema > should render a form unique 1`] = `
class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
for="_r_31_-form-item"
>
- Group by columns
+ Columns
</label>
<div
aria-describedby="_r_31_-form-item-description"
@@ -1366,7 +1593,7 @@ exports[`renderZodSchema > should render a form unique 1`] = `
<span
class="truncate flex-1 min-w-0"
>
- Select group by columns
+ Select columns
</span>
<svg
aria-hidden="true"
@@ -1451,7 +1678,7 @@ exports[`renders custom forms column_id 1`] = `
>
<label
class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
- for="_r_3a_-form-item"
+ for="_r_3p_-form-item"
>
Column
</label>
@@ -1460,7 +1687,7 @@ exports[`renders custom forms column_id 1`] = `
>
<button
aria-autocomplete="none"
- aria-controls="radix-_r_3b_"
+ aria-controls="radix-_r_3q_"
aria-expanded="false"
class="flex h-6 w-fit items-center justify-between rounded-sm bg-background px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground focus:outline-hidden focus:ring-1 focus:ring-ring disabled:cursor-not-allowed disabled:opacity-50 cursor-pointer shadow-xs-solid border border-input hover:shadow-sm-solid focus:border-primary focus:shadow-md-solid disabled:hover:shadow-xs-solid mb-0 min-w-[180px]"
data-placeholder=""
@@ -1504,18 +1731,18 @@ exports[`renders custom forms column_id_array 1`] = `
>
<label
class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
- for="_r_3h_-form-item"
+ for="_r_40_-form-item"
>
- Group by columns
+ Columns
</label>
<div
- aria-describedby="_r_3h_-form-item-description"
+ aria-describedby="_r_40_-form-item-description"
aria-invalid="false"
class="relative"
- id="_r_3h_-form-item"
+ id="_r_40_-form-item"
>
<div
- aria-controls="radix-_r_3i_"
+ aria-controls="radix-_r_41_"
aria-expanded="false"
aria-haspopup="dialog"
class="flex h-6 w-fit mb-1 shadow-xs-solid items-center justify-between rounded-sm border border-input bg-transparent px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground hover:shadow-sm-solid focus:outline-hidden focus:ring-1 focus:ring-ring focus:border-primary focus:shadow-md-solid disabled:cursor-not-allowed disabled:opacity-50 min-w-[180px]"
@@ -1525,7 +1752,7 @@ exports[`renders custom forms column_id_array 1`] = `
<span
class="truncate flex-1 min-w-0"
>
- Select group by columns
+ Select columns
</span>
<svg
aria-hidden="true"
@@ -1559,13 +1786,13 @@ exports[`renders custom forms column_id_dot_array 1`] = `
class="flex flex-col gap-1"
>
<div
- aria-describedby="_r_3j_-form-item-description"
+ aria-describedby="_r_42_-form-item-description"
aria-invalid="false"
class="relative"
- id="_r_3j_-form-item"
+ id="_r_42_-form-item"
>
<div
- aria-controls="radix-_r_3k_"
+ aria-controls="radix-_r_43_"
aria-expanded="false"
aria-haspopup="dialog"
class="flex h-6 w-fit mb-1 shadow-xs-solid items-center justify-between rounded-sm border border-input bg-transparent px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground hover:shadow-sm-solid focus:outline-hidden focus:ring-1 focus:ring-ring focus:border-primary focus:shadow-md-solid disabled:cursor-not-allowed disabled:opacity-50 min-w-[180px]"
@@ -1610,7 +1837,7 @@ exports[`renders custom forms column_id_optional 1`] = `
>
<label
class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
- for="_r_3l_-form-item"
+ for="_r_44_-form-item"
>
Column
</label>
@@ -1619,7 +1846,7 @@ exports[`renders custom forms column_id_optional 1`] = `
>
<button
aria-autocomplete="none"
- aria-controls="radix-_r_3m_"
+ aria-controls="radix-_r_45_"
aria-expanded="false"
class="flex h-6 w-fit items-center justify-between rounded-sm bg-background px-2 text-sm font-prose ring-offset-background placeholder:text-muted-foreground focus:outline-hidden focus:ring-1 focus:ring-ring disabled:cursor-not-allowed disabled:opacity-50 cursor-pointer shadow-xs-solid border border-input hover:shadow-sm-solid focus:border-primary focus:shadow-md-solid disabled:hover:shadow-xs-solid mb-0 min-w-[180px]"
data-placeholder=""
diff --git a/frontend/src/plugins/impl/data-frames/panel.tsx b/frontend/src/plugins/impl/data-frames/panel.tsx
index d27101d4a1..482ed89d93 100644
--- a/frontend/src/plugins/impl/data-frames/panel.tsx
+++ b/frontend/src/plugins/impl/data-frames/panel.tsx
@@ -17,6 +17,7 @@ import {
PlusIcon,
ShuffleIcon,
SquareMousePointerIcon,
+ Table2Icon,
Trash2Icon,
} from "lucide-react";
import React, {
@@ -394,4 +395,5 @@ const ICONS: Record<TransformType["type"], React.FC<{ className?: string }>> = {
explode_columns: BracketsIcon,
expand_dict: FileJsonIcon,
unique: CopySlashIcon,
+ pivot: Table2Icon,
};
diff --git a/frontend/src/plugins/impl/data-frames/schema.ts b/frontend/src/plugins/impl/data-frames/schema.ts
index 98c38c80fd..f9bd569e89 100644
--- a/frontend/src/plugins/impl/data-frames/schema.ts
+++ b/frontend/src/plugins/impl/data-frames/schema.ts
@@ -29,7 +29,7 @@ export const column_id_array = z
.array(column_id.describe(FieldOptions.of({ special: "column_id" })))
.min(1, "At least one column is required")
.default([])
- .describe(FieldOptions.of({ label: "Group by columns", minLength: 1 }));
+ .describe(FieldOptions.of({ label: "Columns", minLength: 1 }));
const ColumnConversionTransformSchema = z
.object({
@@ -107,7 +107,10 @@ const FilterRowsTransformSchema = z.object({
const GroupByTransformSchema = z
.object({
type: z.literal("group_by"),
- column_ids: column_id_array,
+ column_ids: z
+ .array(column_id.describe(FieldOptions.of({ special: "column_id" })))
+ .default([])
+ .describe(FieldOptions.of({ label: "Group by columns", minLength: 1 })),
aggregation_column_ids: z
.array(column_id.describe(FieldOptions.of({ special: "column_id" })))
.default([])
@@ -193,6 +196,25 @@ const UniqueTransformSchema = z
})
.describe(FieldOptions.of({ direction: "row" }));
+const PivotTransformSchema = z
+ .object({
+ type: z.literal("pivot"),
+ column_ids: column_id_array,
+ index_column_ids: z
+ .array(column_id.describe(FieldOptions.of({ special: "column_id" })))
+ .default([])
+ .describe(FieldOptions.of({ label: "Rows" })),
+ value_column_ids: z
+ .array(column_id.describe(FieldOptions.of({ special: "column_id" })))
+ .default([])
+ .describe(FieldOptions.of({ label: "Values", minLength: 1 })),
+ aggregation: z
+ .enum(AGGREGATION_FNS)
+ .default("sum")
+ .describe(FieldOptions.of({ label: "Aggregation" })),
+ })
+ .describe(FieldOptions.of({}));
+
export const TransformTypeSchema = z.union([
FilterRowsTransformSchema,
SelectColumnsTransformSchema,
@@ -206,6 +228,7 @@ export const TransformTypeSchema = z.union([
ExplodeColumnsTransformSchema,
ExpandDictTransformSchema,
UniqueTransformSchema,
+ PivotTransformSchema,
]);
export type TransformType = z.infer<typeof TransformTypeSchema>;
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/apply.py b/marimo/_plugins/ui/_impl/dataframes/transforms/apply.py
index 113a9ed823..8cedb0318d 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/apply.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/apply.py
@@ -51,6 +51,8 @@ def _handle(df: T, handler: TransformHandler[T], transform: Transform) -> T:
return handler.handle_expand_dict(df, transform)
if transform.type is TransformType.UNIQUE:
return handler.handle_unique(df, transform)
+ if transform.type is TransformType.PIVOT:
+ return handler.handle_pivot(df, transform)
assert_never(transform.type)
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
index de6d4845ea..6e5ef748bb 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
@@ -2,6 +2,7 @@
from __future__ import annotations
import datetime
+from functools import reduce
from typing import TYPE_CHECKING, Any, Callable
import narwhals.stable.v2 as nw
@@ -21,6 +22,7 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (
ExplodeColumnsTransform,
FilterRowsTransform,
GroupByTransform,
+ PivotTransform,
RenameColumnTransform,
SampleRowsTransform,
SelectColumnsTransform,
@@ -370,6 +372,89 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):
)
assert_never(keep)
+ @staticmethod
+ def handle_pivot(df: DataFrame, transform: PivotTransform) -> DataFrame:
+ # Since ibis does not have a native pivot, and pivot is not supported for LazyFrame
+ # we implement it manually
+ # pivot results are also highly inconsistent across backends, so we standardize the output here
+
+ if not transform.index_column_ids and not transform.value_column_ids:
+ raise nw.exceptions.InvalidOperationError(
+ "Pivot transform requires at least one index column and or value column."
+ )
+
+ columns = df.collect_schema().names()
+ if not transform.index_column_ids:
+ index_columns = list(
+ filter(
+ lambda col: col not in transform.column_ids
+ and col not in transform.value_column_ids,
+ columns,
+ )
+ )
+ else:
+ index_columns = transform.index_column_ids
+
+ if not transform.value_column_ids:
+ value_columns = list(
+ filter(
+ lambda col: col not in transform.column_ids
+ and col not in transform.index_column_ids,
+ columns,
+ )
+ )
+ else:
+ value_columns = transform.value_column_ids
+
+ raw_pivot_columns = (
+ df.select(*transform.column_ids)
+ .unique()
+ .sort(by=transform.column_ids)
+ .collect()
+ .rows()
+ )
+
+ dfs = []
+ for raw_pivot_column in raw_pivot_columns:
+ aggs = []
+ mask = reduce(
+ lambda x, y: x & y,
+ [
+ nw.col(on_col) == on_val
+ for on_col, on_val in zip(
+ transform.column_ids, raw_pivot_column
+ )
+ ],
+ )
+ for value_column in value_columns:
+ expr = nw.col(value_column).alias(
+ f"{value_column}_{'_'.join(map(str, raw_pivot_column))}_{transform.aggregation}"
+ )
+ if transform.aggregation == "count":
+ aggs.append(expr.len())
+ elif transform.aggregation == "sum":
+ aggs.append(expr.sum())
+ elif transform.aggregation == "mean":
+ aggs.append(expr.mean())
+ elif transform.aggregation == "median":
+ aggs.append(expr.median())
+ elif transform.aggregation == "min":
+ aggs.append(expr.min())
+ elif transform.aggregation == "max":
+ aggs.append(expr.max())
+ else:
+ raise ValueError(
+ f"Unsupported aggregation function: {transform.aggregation}"
+ )
+ dfs.append(df.filter(mask).group_by(*index_columns).agg(*aggs))
+
+ result = df.select(*index_columns).unique()
+ for df_ in dfs:
+ result = result.join(df_, on=index_columns, how="left")
+ if transform.aggregation in {"count", "sum"}:
+ result = result.select(nw.all().fill_null(0))
+ return result.sort(by=index_columns)
+
@staticmethod
def as_python_code(
df: DataFrame,
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
index e4768735e8..17e9497889 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
@@ -217,6 +217,52 @@ def python_print_pandas(
column_ids = transform.column_ids
return f"{df_name}.drop_duplicates({_list_of_strings(column_ids)}, keep={_as_literal(transform.keep)})"
+ elif transform.type == TransformType.PIVOT:
+ if not transform.index_column_ids:
+ index_columns = _list_of_strings(
+ list(
+ filter(
+ lambda col: col not in transform.column_ids
+ and col not in transform.value_column_ids,
+ all_columns,
+ )
+ )
+ )
+ else:
+ index_columns = _list_of_strings(transform.index_column_ids)
+
+ if not transform.value_column_ids:
+ value_columns = _list_of_strings(
+ list(
+ filter(
+ lambda col: col not in transform.column_ids
+ and col not in transform.index_column_ids,
+ all_columns,
+ )
+ )
+ )
+ else:
+ value_columns = _list_of_strings(transform.value_column_ids)
+ column_ids = transform.column_ids
+ agg_func = transform.aggregation
+
+ args = _args_list(
+ f"index={index_columns}",
+ f"columns={_list_of_strings(column_ids)}",
+ f"values={value_columns}",
+ f"aggfunc={_as_literal(agg_func)}",
+ "sort=False",
+ )
+ pivot_code = f"{df_name}.pivot_table({args}).sort_index(axis=0)"
+ flatten_columns_code = (
+ f"{df_name}.columns = ["
+ f"f\"{{'_'.join(map(str, col)).strip()}}_{agg_func}\" "
+ f'if isinstance(col, tuple) else f"{{col}}_{agg_func}" '
+ f"for col in {df_name}.columns]"
+ )
+ reset_index_code = f"{df_name} = {df_name}.reset_index()"
+ return f"{pivot_code}\n{flatten_columns_code}\n{reset_index_code}"
+
assert_never(transform.type)
@@ -378,6 +424,56 @@ def python_print_polars(
column_ids = transform.column_ids
return f"{df_name}.unique(subset={_list_of_strings(column_ids)}, keep={_as_literal(transform.keep)})" # noqa: E501
+ elif transform.type == TransformType.PIVOT:
+ if not transform.index_column_ids:
+ index_column_ids = _list_of_strings(
+ list(
+ filter(
+ lambda col: col not in transform.column_ids
+ and col not in transform.value_column_ids,
+ all_columns,
+ )
+ )
+ )
+ else:
+ index_column_ids = _list_of_strings(transform.index_column_ids)
+
+ if not transform.value_column_ids:
+ value_column_ids = _list_of_strings(
+ list(
+ filter(
+ lambda col: col not in transform.column_ids
+ and col not in transform.index_column_ids,
+ all_columns,
+ )
+ )
+ )
+ else:
+ value_column_ids = _list_of_strings(transform.value_column_ids)
+
+ args = _args_list(
+ f"on={_list_of_strings(transform.column_ids)}",
+ f"index={index_column_ids}",
+ f"values={value_column_ids}",
+ f"aggregate_function={_as_literal(transform.aggregation) if transform.aggregation != 'count' else _as_literal('len')}",
+ )
+ fill_null_code = (
+ ".select(pl.all().fill_null(0))"
+ if transform.aggregation in ["count", "sum"]
+ else ""
+ ) # noqa: E501
+ pivot_code = f"{df_name}{fill_null_code}.pivot({args}).sort(by={index_column_ids})" # noqa: E501
+ lambda_code = (
+ f'lambda col,replacements=replacements: f"{transform.value_column_ids[0]}_{{col.translate(replacements)}}_{transform.aggregation}"' # noqa: E501
+ if len(transform.value_column_ids) == 1
+ else f"lambda col, replacements=replacements: f'{{col.translate(replacements)}}_{transform.aggregation}'"
+ ) + f" if col not in {index_column_ids} else col"
+ rename_code = (
+ 'replacements = str.maketrans({"{": "", "}": "", \'"\': "", ",": "_"})\n'
+ f"{df_name} = {df_name}.rename({lambda_code})"
+ )
+ return f"{pivot_code}\n{rename_code}"
+
assert_never(transform.type)
@@ -508,6 +604,48 @@ def python_print_ibis(
column_ids = transform.column_ids
return f"{df_name}.distinct(on={_list_of_strings(column_ids)}, keep={_as_literal(transform.keep)})" # noqa: E501
+ elif transform.type == TransformType.PIVOT:
+ if not transform.index_column_ids:
+ index_column_ids = _list_of_strings(
+ list(
+ filter(
+ lambda col: col not in transform.column_ids
+ and col not in transform.value_column_ids,
+ all_columns,
+ )
+ )
+ )
+ else:
+ index_column_ids = _list_of_strings(transform.index_column_ids)
+
+ if not transform.value_column_ids:
+ value_column_ids = _list_of_strings(
+ list(
+ filter(
+ lambda col: col not in transform.column_ids
+ and col not in transform.index_column_ids,
+ all_columns,
+ )
+ )
+ )
+ else:
+ value_column_ids = _list_of_strings(transform.value_column_ids)
+
+ column_ids = transform.column_ids
+ agg_func = transform.aggregation
+
+ pivot_code = (
+ f"{df_name}.pivot_wider("
+ f"names_from={_list_of_strings(column_ids)}, "
+ f"id_cols={_list_of_strings(index_column_ids)}, "
+ f"values_from={_list_of_strings(value_column_ids)}, "
+ f"names_prefix={_as_literal(value_column_ids[0]) if len(value_column_ids) == 1 else _as_literal('')}, "
+ f"values_agg={_as_literal(agg_func)})"
+ )
+
+ rename_code = f'{df_name} = {df_name}.rename(**{{f"{{col}}_{agg_func}": col for col in {df_name}.columns if col not in {_list_of_strings(index_column_ids)}}})' # noqa: E501
+ return f"{pivot_code}\n{rename_code}"
+
assert_never(transform.type)
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/types.py b/marimo/_plugins/ui/_impl/dataframes/transforms/types.py
index 754e72c506..e1bb565589 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/types.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/types.py
@@ -66,6 +66,7 @@ class TransformType(Enum):
EXPLODE_COLUMNS = "explode_columns"
EXPAND_DICT = "expand_dict"
UNIQUE = "unique"
+ PIVOT = "pivot"
@dataclass(frozen=True)
@@ -176,10 +177,20 @@ class UniqueTransform:
keep: UniqueKeep
+@dataclass
+class PivotTransform:
+ type: Literal[TransformType.PIVOT]
+ column_ids: ColumnIds
+ index_column_ids: ColumnIds
+ value_column_ids: ColumnIds
+ aggregation: Aggregation
+
+
Transform = Union[
AggregateTransform,
ColumnConversionTransform,
FilterRowsTransform,
+ PivotTransform,
GroupByTransform,
RenameColumnTransform,
SelectColumnsTransform,
@@ -263,6 +274,11 @@ class TransformHandler(abc.ABC, Generic[T]):
def handle_unique(df: T, transform: UniqueTransform) -> T:
raise NotImplementedError
+ @staticmethod
+ @abc.abstractmethod
+ def handle_pivot(df: T, transform: PivotTransform) -> T:
+ raise NotImplementedError
+
@staticmethod
def as_python_code(
df: T, df_name: str, columns: list[str], transforms: list[Transform]
diff --git a/marimo/_smoke_tests/issues/7345-dataframe-pivot-transform.py b/marimo/_smoke_tests/issues/7345-dataframe-pivot-transform.py
new file mode 100644
index 0000000000..98af96c558
--- /dev/null
+++ b/marimo/_smoke_tests/issues/7345-dataframe-pivot-transform.py
@@ -0,0 +1,108 @@
+import marimo
+
+__generated_with = "0.18.4"
+app = marimo.App(width="columns")
+
+
+def _():
+ import marimo as mo
+ import polars as pl
+ import pandas as pd
+ import ibis as ib
+
+ from vega_datasets import data
+ return data, ib, mo, pd, pl
+
+
+def _(data, ib, pd, pl):
+ cars = data.cars()
+ df_pandas = pd.DataFrame(cars)
+ df_polars = pl.DataFrame(cars)
+ df_ibis = ib.memtable(cars)
+ return df_ibis, df_pandas, df_polars
+
+
+def _(df_pandas, mo):
+ mo.ui.dataframe(df_pandas)
+ return
+
+
+def _(df_polars, mo):
+ mo.ui.dataframe(df_polars)
+ return
+
+
+def _(df_ibis, mo):
+ mo.ui.dataframe(df_ibis)
+ return
+
+
+def _(df_pandas):
+ df_pandas_next = df_pandas
+ df_pandas_next = df_pandas_next.pivot_table(
+ index=["Year"],
+ columns=["Origin"],
+ values=["Acceleration"],
+ aggfunc="mean",
+ sort=False,
+ fill_value=None,
+ ).sort_index(axis=0)
+ df_pandas_next.columns = [
+ f"{'_'.join(map(str, col)).strip()}_mean"
+ if isinstance(col, tuple)
+ else f"{col}_mean"
+ for col in df_pandas_next.columns
+ ]
+ df_pandas_next = df_pandas_next.reset_index()
+ df_pandas_next
+ return
+
+
+def _(df_polars):
+ df_polars_next = df_polars
+ df_polars_next = df_polars_next.pivot(
+ on=["Origin"],
+ index=["Year"],
+ values=["Acceleration"],
+ aggregate_function="mean",
+ ).sort(["Year"])
+ replacements = str.maketrans({"{": "", "}": "", '"': "", ",": "_"})
+ df_polars_next = df_polars_next.rename(
+ lambda col: f"Acceleration_{col.translate(replacements)}_mean"
+ if col not in ["Year"]
+ else col
+ )
+ df_polars_next
+ return
+
+
+def _(df_ibis):
+ df_ibis_next = df_ibis
+ df_ibis_next = df_ibis_next.pivot_wider(
+ names_from=["Origin"],
+ id_cols=["Year"],
+ values_from=["Acceleration"],
+ names_prefix="Acceleration",
+ values_agg="mean",
+ )
+ df_ibis_next = df_ibis_next.rename(
+ **{
+ f"{col}_mean": col
+ for col in df_ibis_next.columns
+ if col not in ["Year"]
+ }
+ )
+ df_ibis_next.to_polars()
+ return
+
+
+if __name__ == "__main__":
+ app.run()
diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.py
index 32dabb4757..b9090957e7 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py
@@ -23,6 +23,7 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (
ExplodeColumnsTransform,
FilterRowsTransform,
GroupByTransform,
+ PivotTransform,
RenameColumnTransform,
SampleRowsTransform,
SelectColumnsTransform,
@@ -1184,6 +1185,208 @@ class TestTransformHandler:
)
assert type(result) is type(df)
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("df", "expected", "transform"),
+ [
+ *zip(
+ create_test_dataframes(
+ {
+ "A": [
+ "foo",
+ "foo",
+ "foo",
+ "foo",
+ "foo",
+ "bar",
+ "bar",
+ "bar",
+ "bar",
+ ],
+ "B": [
+ "one",
+ "one",
+ "one",
+ "two",
+ "two",
+ "one",
+ "one",
+ "two",
+ "two",
+ ],
+ "C": [
+ "small",
+ "large",
+ "large",
+ "small",
+ "small",
+ "large",
+ "small",
+ "small",
+ "large",
+ ],
+ "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+ }
+ ),
+ create_test_dataframes(
+ {
+ "B": ["one", "two"],
+ "D_foo_sum": [5, 6],
+ "D_bar_sum": [9, 13],
+ }
+ ),
+ [
+ PivotTransform(
+ type=TransformType.PIVOT,
+ column_ids=["A"],
+ index_column_ids=["B"],
+ value_column_ids=["D"],
+ aggregation="sum",
+ )
+ ],
+ ),
+ *zip(
+ create_test_dataframes(
+ {
+ "A": [
+ "foo",
+ "foo",
+ "foo",
+ "foo",
+ "foo",
+ "bar",
+ "bar",
+ "bar",
+ "bar",
+ ],
+ "B": [
+ "one",
+ "one",
+ "one",
+ "two",
+ "two",
+ "one",
+ "one",
+ "two",
+ "two",
+ ],
+ "C": [
+ "small",
+ "large",
+ "large",
+ "small",
+ "small",
+ "large",
+ "small",
+ "small",
+ "large",
+ ],
+ "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+ }
+ ),
+ create_test_dataframes(
+ {
+ "B": ["one", "one", "two", "two"],
+ "C": ["large", "small", "large", "small"],
+ "D_bar_sum": [4, 5, 7, 6],
+ "D_foo_sum": [4, 1, None, 6],
+ }
+ ),
+ [
+ PivotTransform(
+ type=TransformType.PIVOT,
+ column_ids=["A"],
+ index_column_ids=["B", "C"],
+ value_column_ids=["D"],
+ aggregation="sum",
+ )
+ ],
+ ),
+ *zip(
+ create_test_dataframes(
+ {
+ "A": [
+ "foo",
+ "foo",
+ "bar",
+ "bar",
+ ],
+ "B": [
+ "one",
+ "two",
+ "one",
+ "two",
+ ],
+ "C": [
+ "small",
+ "large",
+ "large",
+ "small",
+ ],
+ "D": [1, 2, 3, 4],
+ }
+ ),
+ create_test_dataframes(
+ {
+ "B": ["one", "one", "two", "two"],
+ "C": ["large", "small", "large", "small"],
+ "D_bar_sum": [3, None, None, 4],
+ "D_foo_sum": [None, 1, 2, None],
+ }
+ ),
+ [
+ PivotTransform(
+ type=TransformType.PIVOT,
+ column_ids=["A"],
+ index_column_ids=[],
+ value_column_ids=["D"],
+ aggregation="sum",
+ )
+ ],
+ ),
+ *zip(
+ create_test_dataframes(
+ {
+ "A": [
+ "foo",
+ "foo",
+ "bar",
+ "bar",
+ ],
+ "B": [
+ "one",
+ "two",
+ "one",
+ "two",
+ ],
+ "D": [1, 2, 3, 4],
+ }
+ ),
+ create_test_dataframes(
+ {
+ "B": ["one", "two"],
+ "D_bar_sum": [3, 4],
+ "D_foo_sum": [1, 2],
+ }
+ ),
+ [
+ PivotTransform(
+ type=TransformType.PIVOT,
+ column_ids=["A"],
+ index_column_ids=["B"],
+ value_column_ids=[],
+ aggregation="sum",
+ )
+ ],
+ ),
+ ],
+ )
+ def test_handle_pivot(
+ df: DataFrameType, expected: DataFrameType, transform: PivotTransform
+ ) -> None:
+ result = apply(df, transform)
+ assert_frame_equal_with_nans(result, expected)
+
@staticmethod
@pytest.mark.parametrize(
("df", "expected", "expected2"),
diff --git a/tests/_plugins/ui/_impl/dataframes/test_print_code.py b/tests/_plugins/ui/_impl/dataframes/test_print_code.py
index addf78d18b..307dcd58e2 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_print_code.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_print_code.py
@@ -30,6 +30,7 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (
ExplodeColumnsTransform,
FilterRowsTransform,
GroupByTransform,
+ PivotTransform,
RenameColumnTransform,
SampleRowsTransform,
SelectColumnsTransform,
@@ -75,10 +76,14 @@ def create_transform_strategy(
list_column_id: Optional[st.SearchStrategy[str | int] | None] = None,
df_size: int = 3,
) -> st.SearchStrategy[Transform]:
- column_ids = st.lists(column_id, min_size=1)
+ column_ids = st.lists(column_id, min_size=1, unique=True)
aggregation_column_ids = st.one_of(
st.just([]),
- st.lists(column_id, min_size=1),
+ st.lists(column_id, min_size=1, unique=True),
+ )
+ index_column_ids = st.one_of(
+ st.just([]),
+ st.lists(column_id, min_size=1, unique=True),
)
if string_column_id is None:
@@ -220,6 +225,15 @@ def create_transform_strategy(
column_id=column_id,
)
+ pivot_transform_strategy = st.builds(
+ PivotTransform,
+ type=st.just(TransformType.PIVOT),
+ column_ids=column_ids,
+ index_column_ids=index_column_ids,
+ aggregation=aggregation,
+ value_column_ids=aggregation_column_ids,
+ )
+
# Combine all transform strategies
transform_strategy = st.one_of(
column_conversion_transform_strategy,
@@ -233,6 +247,7 @@ def create_transform_strategy(
sample_rows_transform_strategy,
explode_columns_transform_strategy,
expand_dict_transform_strategy,
+ pivot_transform_strategy,
)
return transform_strategy
@@ -366,6 +381,42 @@ def test_print_code_result_matches_actual_transform_pandas(
# Ignore groupby mean
if transform.type == TransformType.GROUP_BY:
assume(transform.aggregation != "mean")
+ assume("dict" not in transform.aggregation_column_ids)
+
+ if transform.type == TransformType.PIVOT:
+ assume(
+ (
+ set(transform.column_ids) & set(transform.index_column_ids)
+ == set()
+ )
+ and (
+ set(transform.column_ids) & set(transform.value_column_ids)
+ == set()
+ )
+ and (
+ set(transform.index_column_ids)
+ & set(transform.value_column_ids)
+ == set()
+ )
+ )
+ assume(
+ not any(
+ column_id in {"lists", "dicts", "datetimes"}
+ for column_id in transform.column_ids
+ )
+ )
+ assume(
+ not any(
+ column_id in {"lists", "dicts", "datetimes"}
+ for column_id in transform.index_column_ids
+ )
+ )
+ assume(
+ not any(
+ column_id in {"lists", "dicts", "datetimes"}
+ for column_id in transform.value_column_ids
+ )
+ )
# Pandas
pandas_code = python_print_transforms(
@@ -423,6 +474,11 @@ def test_print_code_result_matches_actual_transform_pandas(
if isinstance(real_result, pd.Series):
real_result = real_result.to_frame()
+ # For pivot transform the column order can be different, enforce column order by sorting.
+ if transform.type == TransformType.PIVOT:
+ code_result = code_result.loc[:, list(sorted(code_result.columns))]
+ real_result = real_result.loc[:, list(sorted(real_result.columns))]
+
code_result = cast(pd.DataFrame, code_result).reset_index(drop=True)
real_result = real_result.reset_index(drop=True)
@@ -516,6 +572,40 @@ def test_print_code_result_matches_actual_transform_polars(
# TODO: unimplemented
if transform.type == TransformType.AGGREGATE:
assume(False)
+ if transform.type == TransformType.PIVOT:
+ assume(
+ (
+ set(transform.column_ids) & set(transform.index_column_ids)
+ == set()
+ )
+ and (
+ set(transform.column_ids) & set(transform.value_column_ids)
+ == set()
+ )
+ and (
+ set(transform.index_column_ids)
+ & set(transform.value_column_ids)
+ == set()
+ )
+ )
+ assume(
+ not any(
+ column_id in {"lists", "dicts", "datetimes"}
+ for column_id in transform.column_ids
+ )
+ )
+ assume(
+ not any(
+ column_id in {"lists", "dicts", "datetimes"}
+ for column_id in transform.index_column_ids
+ )
+ )
+ assume(
+ not any(
+ column_id in {"lists", "dicts", "datetimes"}
+ for column_id in transform.value_column_ids
+ )
+ )
# Polars
polars_code = python_print_transforms(
@@ -575,6 +665,16 @@ def test_print_code_result_matches_actual_transform_polars(
code_result = code_result.to_frame()
if isinstance(real_result, pl.Series):
real_result = real_result.to_frame()
+
+ # For pivot transform the column order can be different, enforce column order by sorting.
+ if transform.type == TransformType.PIVOT:
+ code_result = code_result.select(
+ *sorted(code_result.collect_schema().names())
+ )
+ real_result = real_result.select(
+ *sorted(real_result.collect_schema().names())
+ )
+
code_result = cast(pl.DataFrame, code_result)
# Compare column names
assert code_result.columns == real_result.columns
@@ -652,6 +752,41 @@ def test_print_code_result_matches_actual_transform_ibis(
if transform.type == TransformType.COLUMN_CONVERSION:
assume(transform.errors != "ignore")
+ if transform.type == TransformType.PIVOT:
+ assume(
+ (
+ set(transform.column_ids) & set(transform.index_column_ids)
+ == set()
+ )
+ and (
+ set(transform.column_ids) & set(transform.value_column_ids)
+ == set()
+ )
+ and (
+ set(transform.index_column_ids)
+ & set(transform.value_column_ids)
+ == set()
+ )
+ )
+ assume(
+ not any(
+ column_id in {"lists", "dicts", "datetimes"}
+ for column_id in transform.column_ids
+ )
+ )
+ assume(
+ not any(
+ column_id in {"lists", "dicts", "datetimes"}
+ for column_id in transform.index_column_ids
+ )
+ )
+ assume(
+ not any(
+ column_id in {"lists", "dicts", "datetimes"}
+ for column_id in transform.value_column_ids
+ )
+ )
+
try:
nw_df = nw.from_native(my_df).lazy()
result_nw = _apply_transforms(
@@ -681,6 +816,11 @@ def test_print_code_result_matches_actual_transform_ibis(
exec(ibis_code, {}, loc)
code_result = loc.get("my_df_next")
+ # For pivot transform the column order can be different, enforce column order by sorting.
+ if transform.type == TransformType.PIVOT:
+ code_result = code_result.select(*sorted(code_result.columns))
+ real_result = real_result.select(*sorted(real_result.columns))
+
print("code_result", code_result)
print("real_result", real_result)