A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
add not_in filter for tables and dataframes (#7439)
Shahmir Varqhad1e804ea9ea5
diff --git a/frontend/src/components/data-table/context-menu.tsx b/frontend/src/components/data-table/context-menu.tsxindex 070dd73317..689bdc7110 100644--- a/frontend/src/components/data-table/context-menu.tsx+++ b/frontend/src/components/data-table/context-menu.tsx@@ -95,11 +95,11 @@ export const CellContextMenu = <TData,>({const column = cell.column;const canFilter = column.getCanFilter() && column.columnDef.meta?.filterType;- const handleFilterCell = () => {+ const handleFilterCell = (operator: "in" | "not_in") => {column.setFilterValue(Filter.select({options: [cell.getValue()],- operator: "in",+ operator,}),);};@@ -119,10 +119,14 @@ export const CellContextMenu = <TData,>({{canFilter && (<><ContextMenuSeparator />- <ContextMenuItem onClick={handleFilterCell}>+ <ContextMenuItem onClick={() => handleFilterCell("in")}><FilterIcon className="mo-dropdown-icon h-3 w-3" />Filter by this value</ContextMenuItem>+ <ContextMenuItem onClick={() => handleFilterCell("not_in")}>+ <FilterIcon className="mo-dropdown-icon h-3 w-3" />+ Remove rows with this value+ </ContextMenuItem></>)}</ContextMenuContent>diff --git a/frontend/src/components/data-table/filter-pills.tsx b/frontend/src/components/data-table/filter-pills.tsxindex 163f988a8b..bd02883b2c 100644--- a/frontend/src/components/data-table/filter-pills.tsx+++ b/frontend/src/components/data-table/filter-pills.tsx@@ -96,7 +96,8 @@ function formatValue(value: ColumnFilterValue, timeFormatter: DateFormatter) {const stringifiedOptions = value.options.map((o) =>stringifyUnknownValue({ value: o }),);- return `is in [${stringifiedOptions.join(", ")}]`;+ const operator = value.operator === "in" ? "is in" : "not in";+ return `${operator} [${stringifiedOptions.join(", ")}]`;}if (value.type === "text") {return `contains "${value.text}"`;diff --git a/frontend/src/components/data-table/filters.ts b/frontend/src/components/data-table/filters.tsindex 585d21275b..ff437925ca 100644--- a/frontend/src/components/data-table/filters.ts+++ b/frontend/src/components/data-table/filters.ts@@ -7,6 +7,7 @@ import type { ConditionType } from "@/plugins/impl/data-frames/schema";import type { ColumnId } from "@/plugins/impl/data-frames/types";import type { OperatorType } from "@/plugins/impl/data-frames/utils/operators";import { assertNever } from "@/utils/assertNever";+import { Logger } from "@/utils/Logger";declare module "@tanstack/react-table" {//allows us to define custom properties for our columns@@ -192,12 +193,20 @@ export function filterToFilterCondition(}return [];- case "select":+ case "select": {+ let operator = filter.operator;+ if (filter.operator !== "in" && filter.operator !== "not_in") {+ Logger.warn("Invalid operator for select filter", {+ operator: filter.operator,+ });+ operator = "in"; // default to in operator+ }return {column_id: columnId,- operator: "in",+ operator,value: filter.options,};+ }default:assertNever(filter);diff --git a/frontend/src/plugins/impl/data-frames/utils/__tests__/operators.test.ts b/frontend/src/plugins/impl/data-frames/utils/__tests__/operators.test.tsindex 31bbbb259b..564ee493bf 100644--- a/frontend/src/plugins/impl/data-frames/utils/__tests__/operators.test.ts+++ b/frontend/src/plugins/impl/data-frames/utils/__tests__/operators.test.ts@@ -100,12 +100,14 @@ describe("isConditionValueValid", () => {);expect(isConditionValueValid("contains", "test")).toBe(true);expect(isConditionValueValid("in", ["test"])).toBe(true);+ expect(isConditionValueValid("not_in", ["test"])).toBe(true);});it("should return false if the value is not valid according to the schema for the given operator", () => {expect(isConditionValueValid("==", "not a number")).toBe(false);expect(isConditionValueValid("contains", 123)).toBe(false);expect(isConditionValueValid("in", "not an array")).toBe(false);+ expect(isConditionValueValid("not_in", "not an array")).toBe(false);});it("should return true if the operator does not require a value", () => {diff --git a/frontend/src/plugins/impl/data-frames/utils/operators.ts b/frontend/src/plugins/impl/data-frames/utils/operators.tsindex 0e872b773e..1d5781cdd3 100644--- a/frontend/src/plugins/impl/data-frames/utils/operators.ts+++ b/frontend/src/plugins/impl/data-frames/utils/operators.ts@@ -58,6 +58,7 @@ export const STRING_OPERATORS = {starts_with: [Schema.string],ends_with: [Schema.string],in: [Schema.stringMultiColumnValues],+ not_in: [Schema.stringMultiColumnValues],is_null: [],is_not_null: [],};diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.pyindex 9e5e0c71b6..d90b3b130b 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py@@ -223,6 +223,14 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):condition_expr = column.is_in(value) | column.is_null()else:condition_expr = column.is_in(value or [])+ elif condition.operator == "not_in":+ # ~is_in returns null for null values, so we need to explicitly include/exclude nulls+ if value is not None and None in value:+ condition_expr = ~column.is_in(value) & ~column.is_null()+ else:+ condition_expr = (+ ~column.is_in(value or []) | column.is_null()+ )else:assert_never(condition.operator)diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.pyindex 6d80920f07..8a3a171bed 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py@@ -56,8 +56,9 @@ def python_print_pandas(return f"{df_name}[{_as_literal(column_id)}].str.startswith({_as_literal(value)})" # noqa: E501elif operator == "ends_with":return f"{df_name}[{_as_literal(column_id)}].str.endswith({_as_literal(value)})" # noqa: E501- elif operator == "in":- return f"{df_name}[{_as_literal(column_id)}].isin({_list_of_strings(value)})" # noqa: E501+ elif operator == "in" or operator == "not_in":+ result = f"{df_name}[{_as_literal(column_id)}].isin({_list_of_strings(value)})" # noqa: E501+ return result if operator == "in" else f"~{result}"elif operator == "!=":return (f"{df_name}[{_as_literal(column_id)}].ne({_as_literal(value)})"@@ -237,8 +238,9 @@ def python_print_polars(return f"pl.col({_as_literal(column_id)}).str.starts_with({_as_literal(value)})" # noqa: E501elif operator == "ends_with":return f"pl.col({_as_literal(column_id)}).str.ends_with({_as_literal(value)})" # noqa: E501- elif operator == "in":- return f"pl.col({_as_literal(column_id)}).is_in({_list_of_strings(value)})" # noqa: E501+ elif operator == "in" or operator == "not_in":+ result = f"pl.col({_as_literal(column_id)}).is_in({_list_of_strings(value)})" # noqa: E501+ return result if operator == "in" else f"~{result}"elif operator in [">", ">=", "<", "<="]:return f"pl.col({_as_literal(column_id)}) {operator} {_as_literal(value)}" # noqa: E501elif operator == "is_null":@@ -397,8 +399,9 @@ def python_print_ibis(return f"({df_name}[{_as_literal(column_id)}].startswith({_as_literal(value)}))" # noqa: E501elif operator == "ends_with":return f"({df_name}[{_as_literal(column_id)}].endswith({_as_literal(value)}))" # noqa: E501- elif operator == "in":- return f"({df_name}[{_as_literal(column_id)}].isin({_list_of_strings(value)}))" # noqa: E501+ elif operator == "in" or operator == "not_in":+ result = f"({df_name}[{_as_literal(column_id)}].isin({_list_of_strings(value)}))" # noqa: E501+ return result if operator == "in" else f"~{result}"elif operator in [">", ">=", "<", "<="]:return f"({df_name}[{_as_literal(column_id)}] {operator} {_as_literal(value)})" # noqa: E501elif operator == "is_null":diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/types.py b/marimo/_plugins/ui/_impl/dataframes/transforms/types.pyindex 9e21b2ea7d..510f1ba7ed 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/types.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/types.py@@ -40,6 +40,7 @@ Operator = Literal["starts_with","ends_with","in",+ "not_in",]Aggregation = Literal["count",@@ -77,7 +78,7 @@ class Condition:return hash((self.column_id, self.operator, self.value))def __post_init__(self) -> None:- if self.operator == "in":+ if self.operator == "in" or self.operator == "not_in":if isinstance(self.value, list):# Hack to convert to tuple for frozen dataclass# Only tuples can be hashed@@ -86,7 +87,7 @@ class Condition:passelse:raise ValueError(- "value must be a list or tuple for 'in' operator"+ "value must be a list or tuple for 'in' or 'not_in' operator")diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.pyindex ee07711101..bc4350b3b1 100644--- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py+++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py@@ -584,6 +584,111 @@ class TestTransformHandler:result = apply(df, transform)assert_frame_equal(result, expected)+ @staticmethod+ @pytest.mark.parametrize(+ ("df", "expected"),+ list(+ zip(+ create_test_dataframes({"A": [1, 2, 3], "B": [4, 5, 6]}),+ create_test_dataframes({"A": [3], "B": [6]}),+ )+ ),+ )+ def test_filter_rows_not_in_operator(+ df: DataFrameType, expected: DataFrameType+ ) -> None:+ transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[Condition(column_id="A", operator="not_in", value=[1, 2])],+ )+ result = apply(df, transform)+ assert_frame_equal(result, expected)++ @staticmethod+ @pytest.mark.parametrize(+ ("df", "expected"),+ list(+ zip(+ create_test_dataframes(+ {"A": ["foo", "bar", "baz"], "B": [1, 2, 3]}+ ),+ create_test_dataframes({"A": ["baz"], "B": [3]}),+ )+ ),+ )+ def test_filter_rows_not_in_operator_strings(+ df: DataFrameType, expected: DataFrameType+ ) -> None:+ transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(+ column_id="A", operator="not_in", value=["foo", "bar"]+ )+ ],+ )+ result = apply(df, transform)+ assert_frame_equal(result, expected)++ @staticmethod+ @pytest.mark.parametrize(+ ("df", "expected"),+ [+ *zip(+ create_test_dataframes(+ {"A": [1, 2, 3, None], "B": [4, 5, 6, 7]},+ exclude=["ibis"],+ ),+ create_test_dataframes({"A": [3], "B": [6]}, exclude=["ibis"]),+ ),+ ],+ )+ def test_filter_rows_not_in_operator_with_nulls(+ df: DataFrameType, expected: DataFrameType+ ) -> None:+ # not_in with None in value should exclude rows where A is 1, 2, or null+ transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(column_id="A", operator="not_in", value=[1, 2, None])+ ],+ )+ result = apply(df, transform)+ assert_frame_equal(result, expected)++ @staticmethod+ @pytest.mark.parametrize(+ ("df", "expected"),+ [+ *zip(+ create_test_dataframes(+ {"A": [1, 2, 3, None], "B": [4, 5, 6, 7]},+ exclude=["ibis"],+ ),+ create_test_dataframes(+ {"A": [3, None], "B": [6, 7]}, exclude=["ibis"]+ ),+ ),+ ],+ )+ def test_filter_rows_not_in_operator_keep_nulls(+ df: DataFrameType, expected: DataFrameType+ ) -> None:+ # not_in WITHOUT None in value should keep null rows (only exclude 1 and 2)+ transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[Condition(column_id="A", operator="not_in", value=[1, 2])],+ )+ result = apply(df, transform)+ if nw.dependencies.is_pandas_dataframe(result):+ assert_frame_equal_with_nans(result, expected)+ else:+ assert_frame_equal(result, expected)+@staticmethod@pytest.mark.parametrize(("df", "expected"),diff --git a/tests/_plugins/ui/_impl/dataframes/test_print_code.py b/tests/_plugins/ui/_impl/dataframes/test_print_code.pyindex de14f46575..323a006327 100644--- a/tests/_plugins/ui/_impl/dataframes/test_print_code.py+++ b/tests/_plugins/ui/_impl/dataframes/test_print_code.py@@ -131,7 +131,7 @@ def create_transform_strategy(list_condition_strategy = st.builds(Condition,column_id=list_column_id,- operator=st.just("in"),+ operator=st.sampled_from(["in", "not_in"]),value=st.lists(st.one_of(st.text()), min_size=1),)diff --git a/tests/_plugins/ui/_impl/dataframes/test_transforms.py b/tests/_plugins/ui/_impl/dataframes/test_transforms.pyindex 9543ba060e..461d2602c0 100644--- a/tests/_plugins/ui/_impl/dataframes/test_transforms.py+++ b/tests/_plugins/ui/_impl/dataframes/test_transforms.py@@ -2,6 +2,7 @@from __future__ import annotationsfrom marimo._plugins.ui._impl.dataframes.transforms.types import (+ FilterRowsTransform,Transformations,)from marimo._utils.parse_dataclass import parse_raw@@ -34,3 +35,31 @@ def test_parse_transforms() -> None:}result = parse_raw(value, Transformations)assert isinstance(result, Transformations)+++def test_parse_transforms_with_in_operator() -> None:+ def create_transform(operator: str):+ return {+ "transforms": [+ {+ "type": "filter_rows",+ "operation": "keep_rows",+ "where": [+ {+ "column_id": "category",+ "operator": operator,+ "value": ["A", "B", "C"],+ },+ ],+ },+ ]+ }++ for operator in ["in", "not_in"]:+ value = create_transform(operator)+ result = parse_raw(value, Transformations)+ assert isinstance(result, Transformations)+ # Verify the value is converted to tuple for hashability+ transform = result.transforms[0]+ assert isinstance(transform, FilterRowsTransform)+ assert transform.where[0].value == ("A", "B", "C")