A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: search functionality for DataFrames with index columns (#8136)
Myles Scolnick96c824059725
diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.pyindex d19b6171e9..816e5f65ed 100644--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py@@ -35,6 +35,41 @@ if TYPE_CHECKING:from pandas._typing import DtypeObj+def _trivial_range_index(index: pd.Index) -> bool:+ import pandas as pd++ # A trivial index is an unnamed RangeIndex (0, 1, 2, ...)+ return isinstance(index, pd.RangeIndex) and index.name is None+++def _resolve_index_column_conflicts(df: pd.DataFrame) -> pd.DataFrame:+ """Rename index names that conflict with column names by appending '_index'.++ Avoids 'ValueError: cannot insert x, already exists' on reset_index().+ Modifies the DataFrame in-place and returns it.+ """+ import pandas as pd++ index_names = df.index.names+ conflicting_names = set(index_names) & set(df.columns)+ if not conflicting_names:+ return df++ new_names: list[str] = []+ for name in index_names:+ if name in conflicting_names:+ new_names.append(f"{name}_index")+ else:+ new_names.append(str(name))++ if isinstance(df.index, pd.MultiIndex):+ df.index = df.index.set_names(new_names)+ else:+ df.index = df.index.rename(new_names[0])++ return df++def _maybe_convert_geopandas_to_pandas(data: pd.DataFrame) -> pd.DataFrame:# Convert to pandas dataframe since geopandas will fail on# certain operations (like to_json(orient="records"))@@ -156,38 +191,16 @@ class PandasTableManagerFactory(TableManagerFactory):# Only skip reset for unnamed default RangeIndex (0, 1, 2, ...)if isinstance(result.index, pd.MultiIndex) or (isinstance(result.index, pd.Index)- and not (- isinstance(result.index, pd.RangeIndex)- and result.index.name is None- )+ and not _trivial_range_index(result.index)):- index_names = result.index.namesunnamed_indexes = any(idx is None for idx in result.index.names)index_levels = result.index.nlevels- # Check for name conflicts between index names and column names- # to avoid "cannot insert x, already exists" error- conflicting_names = set(index_names) & set(result.columns)- if conflicting_names:- # Create new names, handling None values- new_names: list[str] = []- for name in result.index.names:- if name in conflicting_names:- new_names.append(f"{name}_index")- else:- new_names.append(str(name))-- # Rename the index to avoid conflict- if isinstance(result.index, pd.MultiIndex):- result.index = result.index.set_names(new_names)- else:- result.index = result.index.rename(new_names[0])-- # Update index_names to reflect the rename- index_names = result.index.names+ _resolve_index_column_conflicts(result)+ index_names = result.index.namesresult = result.reset_index()@@ -296,6 +309,42 @@ class PandasTableManagerFactory(TableManagerFactory):self._original_data.index)+ def _has_non_trivial_index(self) -> bool:+ """Check if the DataFrame has a non-trivial index that should be searched."""+ index = self._original_data.index+ return not _trivial_range_index(index)++ def search(self, query: str) -> PandasTableManager:+ # If there's a non-trivial index, include it in the search+ # by resetting the index first+ if self._has_non_trivial_index():+ index = self._original_data.index+ num_levels = index.nlevels+ original_names = list(index.names)++ working = self._original_data.copy()+ _resolve_index_column_conflicts(working)+ df_with_index = working.reset_index()+ manager = PandasTableManager(df_with_index)++ # Get index column names AFTER manager init, since+ # _handle_non_string_column_names may have converted+ # non-string column names to strings+ index_columns = list(+ manager._original_data.columns[:num_levels]+ )++ result = super(PandasTableManager, manager).search(query)+ native_df: pd.DataFrame = nw.to_native(result.data)++ # Restore the original index structure+ native_df = native_df.set_index(index_columns)+ native_df.index.names = original_names+ return PandasTableManager(native_df)+ result = super().search(query)+ native_df = nw.to_native(result.data)+ return PandasTableManager(native_df)+@staticmethoddef is_type(value: Any) -> bool:return isinstance(value, pd.DataFrame)@@ -304,7 +353,7 @@ class PandasTableManagerFactory(TableManagerFactory):self, index: pd.Index[Any]) -> FieldTypes:# Ignore if it's the default index with no name- if index.name is None and isinstance(index, pd.RangeIndex):+ if _trivial_range_index(index):return []if isinstance(index, pd.MultiIndex):diff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.pyindex 38c578c6a0..f649c00b3d 100644--- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py+++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py@@ -930,6 +930,226 @@ class TestPandasTableManager(unittest.TestCase):assert manager.search("bob").get_num_rows() == 0assert manager.search("nan").get_num_rows() == 4+ def test_search_with_index_columns(self) -> None:+ """+ Test that search works on index columns (e.g., from value_counts()).+ See: https://github.com/marimo-team/marimo/issues/7945+ """+ # Test with value_counts() - the exact case from the issue+ df = pd.DataFrame({"name": ["Alice", "Bob", "Alice", "Charlie"]})+ value_counts_df = df["name"].value_counts().to_frame()+ manager = self.factory.create()(value_counts_df)++ # Search should find values in the index+ assert manager.search("Alice").get_num_rows() == 1+ assert manager.search("Bob").get_num_rows() == 1+ assert manager.search("Dave").get_num_rows() == 0+ # Case insensitive+ assert manager.search("alice").get_num_rows() == 1+ # Partial match+ assert manager.search("li").get_num_rows() == 2 # Alice and Charlie++ # Test with a named index+ df_with_index = pd.DataFrame(+ {"value": [10, 20, 30]},+ index=pd.Index(["foo", "bars", "baz"], name="label"),+ )+ manager2 = self.factory.create()(df_with_index)+ assert manager2.search("foo").get_num_rows() == 1+ assert manager2.search("b").get_num_rows() == 2 # bar and baz++ # Test with MultiIndex+ df_multi = pd.DataFrame(+ {"value": [1, 2, 3, 4]},+ index=pd.MultiIndex.from_tuples(+ [("A", "x"), ("A", "y"), ("B", "x"), ("B", "y")],+ names=["letter", "symbol"],+ ),+ )+ manager3 = self.factory.create()(df_multi)+ assert manager3.search("A").get_num_rows() == 2+ assert manager3.search("x").get_num_rows() == 2++ def test_search_preserves_index_structure(self) -> None:+ """+ Test that search preserves the index structure of the DataFrame.+ The result should have the same index type and names as the original.+ """+ # Test with unnamed string index+ df_unnamed = pd.DataFrame(+ {"value": [10, 20, 30]},+ index=pd.Index(["foo", "bars", "bare"]),+ )+ manager = self.factory.create()(df_unnamed)+ result = manager.search("bar")+ result_df = nw.to_native(result.data)++ # Index should be preserved (not converted to a column)+ expected = pd.DataFrame(+ {"value": [20, 30]},+ index=pd.Index(["bars", "bare"]),+ )+ assert_frame_equal(result_df, expected)++ def test_search_preserves_named_index(self) -> None:+ """+ Test that search preserves named index.+ """+ df_named = pd.DataFrame(+ {"value": [10, 20, 30]},+ index=pd.Index(["foo", "bars", "bare"], name="label"),+ )+ manager = self.factory.create()(df_named)+ result = manager.search("bar")+ result_df = nw.to_native(result.data)++ # Index name should be preserved+ expected = pd.DataFrame(+ {"value": [20, 30]},+ index=pd.Index(["bars", "bare"], name="label"),+ )+ assert_frame_equal(result_df, expected)++ def test_search_preserves_multiindex(self) -> None:+ """+ Test that search preserves MultiIndex structure.+ """+ df_multi = pd.DataFrame(+ {"value": [1, 2, 3, 4]},+ index=pd.MultiIndex.from_tuples(+ [("A", "x"), ("A", "y"), ("B", "x"), ("B", "y")],+ names=["letter", "symbol"],+ ),+ )+ manager = self.factory.create()(df_multi)+ result = manager.search("A")+ result_df = nw.to_native(result.data)++ # MultiIndex should be preserved+ expected = pd.DataFrame(+ {"value": [1, 2]},+ index=pd.MultiIndex.from_tuples(+ [("A", "x"), ("A", "y")],+ names=["letter", "symbol"],+ ),+ )+ assert_frame_equal(result_df, expected)++ def test_search_preserves_unnamed_multiindex(self) -> None:+ """+ Test that search preserves unnamed MultiIndex structure.+ """+ df_multi_unnamed = pd.DataFrame(+ {"value": [1, 2, 3, 4]},+ index=pd.MultiIndex.from_tuples(+ [("A", "x"), ("A", "y"), ("B", "x"), ("B", "y")],+ ),+ )+ manager = self.factory.create()(df_multi_unnamed)+ result = manager.search("B")+ result_df = nw.to_native(result.data)++ # MultiIndex should be preserved with None names+ expected = pd.DataFrame(+ {"value": [3, 4]},+ index=pd.MultiIndex.from_tuples(+ [("B", "x"), ("B", "y")],+ ),+ )+ assert_frame_equal(result_df, expected)++ def test_search_preserves_datetime_index(self) -> None:+ """+ Test that search preserves DatetimeIndex.+ """+ df_datetime = pd.DataFrame(+ {"value": ["jan", "feb", "march"]},+ index=pd.to_datetime(["2021-01-01", "2021-02-01", "2021-03-01"]),+ )+ manager = self.factory.create()(df_datetime)+ result = manager.search("jan")+ result_df = nw.to_native(result.data)++ # DatetimeIndex should be preserved+ expected = pd.DataFrame(+ {"value": ["jan"]},+ index=pd.to_datetime(["2021-01-01"]),+ )+ assert_frame_equal(result_df, expected)++ def test_search_with_trivial_index_no_change(self) -> None:+ """+ Test that search with trivial RangeIndex doesn't add extra columns.+ """+ df_trivial = pd.DataFrame(+ {"A": [1, 2, 3], "B": ["foo", "bars", "bare"]},+ )+ manager = self.factory.create()(df_trivial)+ result = manager.search("bar")+ result_df = nw.to_native(result.data)++ # Should have no named index, no extra columns+ # Note: filtering preserves original row indices (1, 2) not (0, 1)+ expected = pd.DataFrame(+ {"A": [2, 3], "B": ["bars", "bare"]},+ index=pd.Index([1, 2]),+ )+ assert_frame_equal(result_df, expected)++ def test_search_preserves_integer_index(self) -> None:+ """+ Test that search preserves integer index (non-RangeIndex).+ This tests the case where index values are integers but not a RangeIndex.+ """+ df_int_index = pd.DataFrame(+ {"value": ["foo", "bars", "bare"]},+ index=pd.Index([100, 200, 300]),+ )+ manager = self.factory.create()(df_int_index)+ result = manager.search("bar")+ result_df = nw.to_native(result.data)++ expected = pd.DataFrame(+ {"value": ["bars", "bare"]},+ index=pd.Index([200, 300]),+ )+ assert_frame_equal(result_df, expected)++ def test_search_preserves_multiindex_with_integer_level_names(+ self,+ ) -> None:+ """+ Test that search works with MultiIndex that has integer level names.+ This tests the bug where non-string column names cause set_index to fail.++ When reset_index() creates columns from index levels, unnamed levels get+ default names like 'level_0', 'level_1'. But if the index had non-string+ names (e.g., integers), those become column names. The+ _handle_non_string_column_names method converts these to strings,+ but we need to use the converted names when restoring the index.+ """+ # Create a DataFrame with integer column names that will be used as index+ df = pd.DataFrame(+ {+ 0: ["A", "A", "B", "B"],+ 1: ["x", "y", "x", "y"],+ "value": [1, 2, 3, 4],+ }+ )+ df = df.set_index([0, 1])+ manager = self.factory.create()(df)+ result = manager.search("A")+ result_df = nw.to_native(result.data)++ expected = pd.DataFrame(+ {"value": [1, 2]},+ index=pd.MultiIndex.from_tuples(+ [("A", "x"), ("A", "y")],+ names=[0, 1],+ ),+ )+ assert_frame_equal(result_df, expected)+def test_apply_formatting_does_not_modify_original_data(self) -> None:original_data = self.data.copy()format_mapping = {@@ -1695,3 +1915,29 @@ class TestPandasTableManager(unittest.TestCase):# Polygon geometryassert isinstance(json_data[2]["geometry"], str)assert "POLYGON" in json_data[2]["geometry"]++ def test_search_with_index_column_name_conflict(self) -> None:+ df = pd.DataFrame(+ {"x": [10, 20, 30], "y": ["foo", "bar", "baz"]},+ )+ df.index = pd.Index([1, 2, 3], name="x")+ manager = self.factory.create()(df)+ # Should not raise ValueError: cannot insert x, already exists+ result = manager.search("foo")+ assert result.get_num_rows() == 1+ # Index should be preserved+ assert result._original_data.index.name == "x"++ def test_search_with_multiindex_column_name_conflict(self) -> None:+ df = pd.DataFrame(+ {"x": [10, 20, 30], "y": ["foo", "bar", "baz"]},+ index=pd.MultiIndex.from_tuples(+ [(1, "a"), (2, "b"), (3, "c")], names=["x", "level"]+ ),+ )+ manager = self.factory.create()(df)+ # Should not raise ValueError: cannot insert x, already exists+ result = manager.search("bar")+ assert result.get_num_rows() == 1+ # MultiIndex should be preserved with original names+ assert list(result._original_data.index.names) == ["x", "level"]