Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

add not_in filter for tables and dataframes (#7439)

Shahmir Varqhad1e804ea9ea5
diff --git a/frontend/src/components/data-table/context-menu.tsx b/frontend/src/components/data-table/context-menu.tsx
index 070dd73317..689bdc7110 100644
--- a/frontend/src/components/data-table/context-menu.tsx
+++ b/frontend/src/components/data-table/context-menu.tsx
@@ -95,11 +95,11 @@ export const CellContextMenu = <TData,>({
const column = cell.column;
const canFilter = column.getCanFilter() && column.columnDef.meta?.filterType;
- const handleFilterCell = () => {
+ const handleFilterCell = (operator: "in" | "not_in") => {
column.setFilterValue(
Filter.select({
options: [cell.getValue()],
- operator: "in",
+ operator,
}),
);
};
@@ -119,10 +119,14 @@ export const CellContextMenu = <TData,>({
{canFilter && (
<>
<ContextMenuSeparator />
- <ContextMenuItem onClick={handleFilterCell}>
+ <ContextMenuItem onClick={() => handleFilterCell("in")}>
<FilterIcon className="mo-dropdown-icon h-3 w-3" />
Filter by this value
</ContextMenuItem>
+ <ContextMenuItem onClick={() => handleFilterCell("not_in")}>
+ <FilterIcon className="mo-dropdown-icon h-3 w-3" />
+ Remove rows with this value
+ </ContextMenuItem>
</>
)}
</ContextMenuContent>
diff --git a/frontend/src/components/data-table/filter-pills.tsx b/frontend/src/components/data-table/filter-pills.tsx
index 163f988a8b..bd02883b2c 100644
--- a/frontend/src/components/data-table/filter-pills.tsx
+++ b/frontend/src/components/data-table/filter-pills.tsx
@@ -96,7 +96,8 @@ function formatValue(value: ColumnFilterValue, timeFormatter: DateFormatter) {
const stringifiedOptions = value.options.map((o) =>
stringifyUnknownValue({ value: o }),
);
- return `is in [${stringifiedOptions.join(", ")}]`;
+ const operator = value.operator === "in" ? "is in" : "not in";
+ return `${operator} [${stringifiedOptions.join(", ")}]`;
}
if (value.type === "text") {
return `contains "${value.text}"`;
diff --git a/frontend/src/components/data-table/filters.ts b/frontend/src/components/data-table/filters.ts
index 585d21275b..ff437925ca 100644
--- a/frontend/src/components/data-table/filters.ts
+++ b/frontend/src/components/data-table/filters.ts
@@ -7,6 +7,7 @@ import type { ConditionType } from "@/plugins/impl/data-frames/schema";
import type { ColumnId } from "@/plugins/impl/data-frames/types";
import type { OperatorType } from "@/plugins/impl/data-frames/utils/operators";
import { assertNever } from "@/utils/assertNever";
+import { Logger } from "@/utils/Logger";
declare module "@tanstack/react-table" {
//allows us to define custom properties for our columns
@@ -192,12 +193,20 @@ export function filterToFilterCondition(
}
return [];
- case "select":
+ case "select": {
+ let operator = filter.operator;
+ if (filter.operator !== "in" && filter.operator !== "not_in") {
+ Logger.warn("Invalid operator for select filter", {
+ operator: filter.operator,
+ });
+ operator = "in"; // default to in operator
+ }
return {
column_id: columnId,
- operator: "in",
+ operator,
value: filter.options,
};
+ }
default:
assertNever(filter);
diff --git a/frontend/src/plugins/impl/data-frames/utils/__tests__/operators.test.ts b/frontend/src/plugins/impl/data-frames/utils/__tests__/operators.test.ts
index 31bbbb259b..564ee493bf 100644
--- a/frontend/src/plugins/impl/data-frames/utils/__tests__/operators.test.ts
+++ b/frontend/src/plugins/impl/data-frames/utils/__tests__/operators.test.ts
@@ -100,12 +100,14 @@ describe("isConditionValueValid", () => {
);
expect(isConditionValueValid("contains", "test")).toBe(true);
expect(isConditionValueValid("in", ["test"])).toBe(true);
+ expect(isConditionValueValid("not_in", ["test"])).toBe(true);
});
it("should return false if the value is not valid according to the schema for the given operator", () => {
expect(isConditionValueValid("==", "not a number")).toBe(false);
expect(isConditionValueValid("contains", 123)).toBe(false);
expect(isConditionValueValid("in", "not an array")).toBe(false);
+ expect(isConditionValueValid("not_in", "not an array")).toBe(false);
});
it("should return true if the operator does not require a value", () => {
diff --git a/frontend/src/plugins/impl/data-frames/utils/operators.ts b/frontend/src/plugins/impl/data-frames/utils/operators.ts
index 0e872b773e..1d5781cdd3 100644
--- a/frontend/src/plugins/impl/data-frames/utils/operators.ts
+++ b/frontend/src/plugins/impl/data-frames/utils/operators.ts
@@ -58,6 +58,7 @@ export const STRING_OPERATORS = {
starts_with: [Schema.string],
ends_with: [Schema.string],
in: [Schema.stringMultiColumnValues],
+ not_in: [Schema.stringMultiColumnValues],
is_null: [],
is_not_null: [],
};
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
index 9e5e0c71b6..d90b3b130b 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
@@ -223,6 +223,14 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):
condition_expr = column.is_in(value) | column.is_null()
else:
condition_expr = column.is_in(value or [])
+ elif condition.operator == "not_in":
+ # ~is_in returns null for null values, so we need to explicitly include/exclude nulls
+ if value is not None and None in value:
+ condition_expr = ~column.is_in(value) & ~column.is_null()
+ else:
+ condition_expr = (
+ ~column.is_in(value or []) | column.is_null()
+ )
else:
assert_never(condition.operator)
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
index 6d80920f07..8a3a171bed 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
@@ -56,8 +56,9 @@ def python_print_pandas(
return f"{df_name}[{_as_literal(column_id)}].str.startswith({_as_literal(value)})" # noqa: E501
elif operator == "ends_with":
return f"{df_name}[{_as_literal(column_id)}].str.endswith({_as_literal(value)})" # noqa: E501
- elif operator == "in":
- return f"{df_name}[{_as_literal(column_id)}].isin({_list_of_strings(value)})" # noqa: E501
+ elif operator == "in" or operator == "not_in":
+ result = f"{df_name}[{_as_literal(column_id)}].isin({_list_of_strings(value)})" # noqa: E501
+ return result if operator == "in" else f"~{result}"
elif operator == "!=":
return (
f"{df_name}[{_as_literal(column_id)}].ne({_as_literal(value)})"
@@ -237,8 +238,9 @@ def python_print_polars(
return f"pl.col({_as_literal(column_id)}).str.starts_with({_as_literal(value)})" # noqa: E501
elif operator == "ends_with":
return f"pl.col({_as_literal(column_id)}).str.ends_with({_as_literal(value)})" # noqa: E501
- elif operator == "in":
- return f"pl.col({_as_literal(column_id)}).is_in({_list_of_strings(value)})" # noqa: E501
+ elif operator == "in" or operator == "not_in":
+ result = f"pl.col({_as_literal(column_id)}).is_in({_list_of_strings(value)})" # noqa: E501
+ return result if operator == "in" else f"~{result}"
elif operator in [">", ">=", "<", "<="]:
return f"pl.col({_as_literal(column_id)}) {operator} {_as_literal(value)}" # noqa: E501
elif operator == "is_null":
@@ -397,8 +399,9 @@ def python_print_ibis(
return f"({df_name}[{_as_literal(column_id)}].startswith({_as_literal(value)}))" # noqa: E501
elif operator == "ends_with":
return f"({df_name}[{_as_literal(column_id)}].endswith({_as_literal(value)}))" # noqa: E501
- elif operator == "in":
- return f"({df_name}[{_as_literal(column_id)}].isin({_list_of_strings(value)}))" # noqa: E501
+ elif operator == "in" or operator == "not_in":
+ result = f"({df_name}[{_as_literal(column_id)}].isin({_list_of_strings(value)}))" # noqa: E501
+ return result if operator == "in" else f"~{result}"
elif operator in [">", ">=", "<", "<="]:
return f"({df_name}[{_as_literal(column_id)}] {operator} {_as_literal(value)})" # noqa: E501
elif operator == "is_null":
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/types.py b/marimo/_plugins/ui/_impl/dataframes/transforms/types.py
index 9e21b2ea7d..510f1ba7ed 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/types.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/types.py
@@ -40,6 +40,7 @@ Operator = Literal[
"starts_with",
"ends_with",
"in",
+ "not_in",
]
Aggregation = Literal[
"count",
@@ -77,7 +78,7 @@ class Condition:
return hash((self.column_id, self.operator, self.value))
def __post_init__(self) -> None:
- if self.operator == "in":
+ if self.operator == "in" or self.operator == "not_in":
if isinstance(self.value, list):
# Hack to convert to tuple for frozen dataclass
# Only tuples can be hashed
@@ -86,7 +87,7 @@ class Condition:
pass
else:
raise ValueError(
- "value must be a list or tuple for 'in' operator"
+ "value must be a list or tuple for 'in' or 'not_in' operator"
)
diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.py
index ee07711101..bc4350b3b1 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py
@@ -584,6 +584,111 @@ class TestTransformHandler:
result = apply(df, transform)
assert_frame_equal(result, expected)
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("df", "expected"),
+ list(
+ zip(
+ create_test_dataframes({"A": [1, 2, 3], "B": [4, 5, 6]}),
+ create_test_dataframes({"A": [3], "B": [6]}),
+ )
+ ),
+ )
+ def test_filter_rows_not_in_operator(
+ df: DataFrameType, expected: DataFrameType
+ ) -> None:
+ transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[Condition(column_id="A", operator="not_in", value=[1, 2])],
+ )
+ result = apply(df, transform)
+ assert_frame_equal(result, expected)
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("df", "expected"),
+ list(
+ zip(
+ create_test_dataframes(
+ {"A": ["foo", "bar", "baz"], "B": [1, 2, 3]}
+ ),
+ create_test_dataframes({"A": ["baz"], "B": [3]}),
+ )
+ ),
+ )
+ def test_filter_rows_not_in_operator_strings(
+ df: DataFrameType, expected: DataFrameType
+ ) -> None:
+ transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(
+ column_id="A", operator="not_in", value=["foo", "bar"]
+ )
+ ],
+ )
+ result = apply(df, transform)
+ assert_frame_equal(result, expected)
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("df", "expected"),
+ [
+ *zip(
+ create_test_dataframes(
+ {"A": [1, 2, 3, None], "B": [4, 5, 6, 7]},
+ exclude=["ibis"],
+ ),
+ create_test_dataframes({"A": [3], "B": [6]}, exclude=["ibis"]),
+ ),
+ ],
+ )
+ def test_filter_rows_not_in_operator_with_nulls(
+ df: DataFrameType, expected: DataFrameType
+ ) -> None:
+ # not_in with None in value should exclude rows where A is 1, 2, or null
+ transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(column_id="A", operator="not_in", value=[1, 2, None])
+ ],
+ )
+ result = apply(df, transform)
+ assert_frame_equal(result, expected)
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("df", "expected"),
+ [
+ *zip(
+ create_test_dataframes(
+ {"A": [1, 2, 3, None], "B": [4, 5, 6, 7]},
+ exclude=["ibis"],
+ ),
+ create_test_dataframes(
+ {"A": [3, None], "B": [6, 7]}, exclude=["ibis"]
+ ),
+ ),
+ ],
+ )
+ def test_filter_rows_not_in_operator_keep_nulls(
+ df: DataFrameType, expected: DataFrameType
+ ) -> None:
+ # not_in WITHOUT None in value should keep null rows (only exclude 1 and 2)
+ transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[Condition(column_id="A", operator="not_in", value=[1, 2])],
+ )
+ result = apply(df, transform)
+ if nw.dependencies.is_pandas_dataframe(result):
+ assert_frame_equal_with_nans(result, expected)
+ else:
+ assert_frame_equal(result, expected)
+
@staticmethod
@pytest.mark.parametrize(
("df", "expected"),
diff --git a/tests/_plugins/ui/_impl/dataframes/test_print_code.py b/tests/_plugins/ui/_impl/dataframes/test_print_code.py
index de14f46575..323a006327 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_print_code.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_print_code.py
@@ -131,7 +131,7 @@ def create_transform_strategy(
list_condition_strategy = st.builds(
Condition,
column_id=list_column_id,
- operator=st.just("in"),
+ operator=st.sampled_from(["in", "not_in"]),
value=st.lists(st.one_of(st.text()), min_size=1),
)
diff --git a/tests/_plugins/ui/_impl/dataframes/test_transforms.py b/tests/_plugins/ui/_impl/dataframes/test_transforms.py
index 9543ba060e..461d2602c0 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_transforms.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_transforms.py
@@ -2,6 +2,7 @@
from __future__ import annotations
from marimo._plugins.ui._impl.dataframes.transforms.types import (
+ FilterRowsTransform,
Transformations,
)
from marimo._utils.parse_dataclass import parse_raw
@@ -34,3 +35,31 @@ def test_parse_transforms() -> None:
}
result = parse_raw(value, Transformations)
assert isinstance(result, Transformations)
+
+
+def test_parse_transforms_with_in_operator() -> None:
+ def create_transform(operator: str):
+ return {
+ "transforms": [
+ {
+ "type": "filter_rows",
+ "operation": "keep_rows",
+ "where": [
+ {
+ "column_id": "category",
+ "operator": operator,
+ "value": ["A", "B", "C"],
+ },
+ ],
+ },
+ ]
+ }
+
+ for operator in ["in", "not_in"]:
+ value = create_transform(operator)
+ result = parse_raw(value, Transformations)
+ assert isinstance(result, Transformations)
+ # Verify the value is converted to tuple for hashability
+ transform = result.transforms[0]
+ assert isinstance(transform, FilterRowsTransform)
+ assert transform.where[0].value == ("A", "B", "C")