Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: search functionality for DataFrames with index columns (#8136)

Myles Scolnick96c824059725
diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.py
index d19b6171e9..816e5f65ed 100644
--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py
+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py
@@ -35,6 +35,41 @@ if TYPE_CHECKING:
from pandas._typing import DtypeObj
+def _trivial_range_index(index: pd.Index) -> bool:
+ import pandas as pd
+
+ # A trivial index is an unnamed RangeIndex (0, 1, 2, ...)
+ return isinstance(index, pd.RangeIndex) and index.name is None
+
+
+def _resolve_index_column_conflicts(df: pd.DataFrame) -> pd.DataFrame:
+ """Rename index names that conflict with column names by appending '_index'.
+
+ Avoids 'ValueError: cannot insert x, already exists' on reset_index().
+ Modifies the DataFrame in-place and returns it.
+ """
+ import pandas as pd
+
+ index_names = df.index.names
+ conflicting_names = set(index_names) & set(df.columns)
+ if not conflicting_names:
+ return df
+
+ new_names: list[str] = []
+ for name in index_names:
+ if name in conflicting_names:
+ new_names.append(f"{name}_index")
+ else:
+ new_names.append(str(name))
+
+ if isinstance(df.index, pd.MultiIndex):
+ df.index = df.index.set_names(new_names)
+ else:
+ df.index = df.index.rename(new_names[0])
+
+ return df
+
+
def _maybe_convert_geopandas_to_pandas(data: pd.DataFrame) -> pd.DataFrame:
# Convert to pandas dataframe since geopandas will fail on
# certain operations (like to_json(orient="records"))
@@ -156,38 +191,16 @@ class PandasTableManagerFactory(TableManagerFactory):
# Only skip reset for unnamed default RangeIndex (0, 1, 2, ...)
if isinstance(result.index, pd.MultiIndex) or (
isinstance(result.index, pd.Index)
- and not (
- isinstance(result.index, pd.RangeIndex)
- and result.index.name is None
- )
+ and not _trivial_range_index(result.index)
):
- index_names = result.index.names
unnamed_indexes = any(
idx is None for idx in result.index.names
)
index_levels = result.index.nlevels
- # Check for name conflicts between index names and column names
- # to avoid "cannot insert x, already exists" error
- conflicting_names = set(index_names) & set(result.columns)
- if conflicting_names:
- # Create new names, handling None values
- new_names: list[str] = []
- for name in result.index.names:
- if name in conflicting_names:
- new_names.append(f"{name}_index")
- else:
- new_names.append(str(name))
-
- # Rename the index to avoid conflict
- if isinstance(result.index, pd.MultiIndex):
- result.index = result.index.set_names(new_names)
- else:
- result.index = result.index.rename(new_names[0])
-
- # Update index_names to reflect the rename
- index_names = result.index.names
+ _resolve_index_column_conflicts(result)
+ index_names = result.index.names
result = result.reset_index()
@@ -296,6 +309,42 @@ class PandasTableManagerFactory(TableManagerFactory):
self._original_data.index
)
+ def _has_non_trivial_index(self) -> bool:
+ """Check if the DataFrame has a non-trivial index that should be searched."""
+ index = self._original_data.index
+ return not _trivial_range_index(index)
+
+ def search(self, query: str) -> PandasTableManager:
+ # If there's a non-trivial index, include it in the search
+ # by resetting the index first
+ if self._has_non_trivial_index():
+ index = self._original_data.index
+ num_levels = index.nlevels
+ original_names = list(index.names)
+
+ working = self._original_data.copy()
+ _resolve_index_column_conflicts(working)
+ df_with_index = working.reset_index()
+ manager = PandasTableManager(df_with_index)
+
+ # Get index column names AFTER manager init, since
+ # _handle_non_string_column_names may have converted
+ # non-string column names to strings
+ index_columns = list(
+ manager._original_data.columns[:num_levels]
+ )
+
+ result = super(PandasTableManager, manager).search(query)
+ native_df: pd.DataFrame = nw.to_native(result.data)
+
+ # Restore the original index structure
+ native_df = native_df.set_index(index_columns)
+ native_df.index.names = original_names
+ return PandasTableManager(native_df)
+ result = super().search(query)
+ native_df = nw.to_native(result.data)
+ return PandasTableManager(native_df)
+
@staticmethod
def is_type(value: Any) -> bool:
return isinstance(value, pd.DataFrame)
@@ -304,7 +353,7 @@ class PandasTableManagerFactory(TableManagerFactory):
self, index: pd.Index[Any]
) -> FieldTypes:
# Ignore if it's the default index with no name
- if index.name is None and isinstance(index, pd.RangeIndex):
+ if _trivial_range_index(index):
return []
if isinstance(index, pd.MultiIndex):
diff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
index 38c578c6a0..f649c00b3d 100644
--- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py
+++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
@@ -930,6 +930,226 @@ class TestPandasTableManager(unittest.TestCase):
assert manager.search("bob").get_num_rows() == 0
assert manager.search("nan").get_num_rows() == 4
+ def test_search_with_index_columns(self) -> None:
+ """
+ Test that search works on index columns (e.g., from value_counts()).
+ See: https://github.com/marimo-team/marimo/issues/7945
+ """
+ # Test with value_counts() - the exact case from the issue
+ df = pd.DataFrame({"name": ["Alice", "Bob", "Alice", "Charlie"]})
+ value_counts_df = df["name"].value_counts().to_frame()
+ manager = self.factory.create()(value_counts_df)
+
+ # Search should find values in the index
+ assert manager.search("Alice").get_num_rows() == 1
+ assert manager.search("Bob").get_num_rows() == 1
+ assert manager.search("Dave").get_num_rows() == 0
+ # Case insensitive
+ assert manager.search("alice").get_num_rows() == 1
+ # Partial match
+ assert manager.search("li").get_num_rows() == 2 # Alice and Charlie
+
+ # Test with a named index
+ df_with_index = pd.DataFrame(
+ {"value": [10, 20, 30]},
+ index=pd.Index(["foo", "bars", "baz"], name="label"),
+ )
+ manager2 = self.factory.create()(df_with_index)
+ assert manager2.search("foo").get_num_rows() == 1
+ assert manager2.search("b").get_num_rows() == 2 # bar and baz
+
+ # Test with MultiIndex
+ df_multi = pd.DataFrame(
+ {"value": [1, 2, 3, 4]},
+ index=pd.MultiIndex.from_tuples(
+ [("A", "x"), ("A", "y"), ("B", "x"), ("B", "y")],
+ names=["letter", "symbol"],
+ ),
+ )
+ manager3 = self.factory.create()(df_multi)
+ assert manager3.search("A").get_num_rows() == 2
+ assert manager3.search("x").get_num_rows() == 2
+
+ def test_search_preserves_index_structure(self) -> None:
+ """
+ Test that search preserves the index structure of the DataFrame.
+ The result should have the same index type and names as the original.
+ """
+ # Test with unnamed string index
+ df_unnamed = pd.DataFrame(
+ {"value": [10, 20, 30]},
+ index=pd.Index(["foo", "bars", "bare"]),
+ )
+ manager = self.factory.create()(df_unnamed)
+ result = manager.search("bar")
+ result_df = nw.to_native(result.data)
+
+ # Index should be preserved (not converted to a column)
+ expected = pd.DataFrame(
+ {"value": [20, 30]},
+ index=pd.Index(["bars", "bare"]),
+ )
+ assert_frame_equal(result_df, expected)
+
+ def test_search_preserves_named_index(self) -> None:
+ """
+ Test that search preserves named index.
+ """
+ df_named = pd.DataFrame(
+ {"value": [10, 20, 30]},
+ index=pd.Index(["foo", "bars", "bare"], name="label"),
+ )
+ manager = self.factory.create()(df_named)
+ result = manager.search("bar")
+ result_df = nw.to_native(result.data)
+
+ # Index name should be preserved
+ expected = pd.DataFrame(
+ {"value": [20, 30]},
+ index=pd.Index(["bars", "bare"], name="label"),
+ )
+ assert_frame_equal(result_df, expected)
+
+ def test_search_preserves_multiindex(self) -> None:
+ """
+ Test that search preserves MultiIndex structure.
+ """
+ df_multi = pd.DataFrame(
+ {"value": [1, 2, 3, 4]},
+ index=pd.MultiIndex.from_tuples(
+ [("A", "x"), ("A", "y"), ("B", "x"), ("B", "y")],
+ names=["letter", "symbol"],
+ ),
+ )
+ manager = self.factory.create()(df_multi)
+ result = manager.search("A")
+ result_df = nw.to_native(result.data)
+
+ # MultiIndex should be preserved
+ expected = pd.DataFrame(
+ {"value": [1, 2]},
+ index=pd.MultiIndex.from_tuples(
+ [("A", "x"), ("A", "y")],
+ names=["letter", "symbol"],
+ ),
+ )
+ assert_frame_equal(result_df, expected)
+
+ def test_search_preserves_unnamed_multiindex(self) -> None:
+ """
+ Test that search preserves unnamed MultiIndex structure.
+ """
+ df_multi_unnamed = pd.DataFrame(
+ {"value": [1, 2, 3, 4]},
+ index=pd.MultiIndex.from_tuples(
+ [("A", "x"), ("A", "y"), ("B", "x"), ("B", "y")],
+ ),
+ )
+ manager = self.factory.create()(df_multi_unnamed)
+ result = manager.search("B")
+ result_df = nw.to_native(result.data)
+
+ # MultiIndex should be preserved with None names
+ expected = pd.DataFrame(
+ {"value": [3, 4]},
+ index=pd.MultiIndex.from_tuples(
+ [("B", "x"), ("B", "y")],
+ ),
+ )
+ assert_frame_equal(result_df, expected)
+
+ def test_search_preserves_datetime_index(self) -> None:
+ """
+ Test that search preserves DatetimeIndex.
+ """
+ df_datetime = pd.DataFrame(
+ {"value": ["jan", "feb", "march"]},
+ index=pd.to_datetime(["2021-01-01", "2021-02-01", "2021-03-01"]),
+ )
+ manager = self.factory.create()(df_datetime)
+ result = manager.search("jan")
+ result_df = nw.to_native(result.data)
+
+ # DatetimeIndex should be preserved
+ expected = pd.DataFrame(
+ {"value": ["jan"]},
+ index=pd.to_datetime(["2021-01-01"]),
+ )
+ assert_frame_equal(result_df, expected)
+
+ def test_search_with_trivial_index_no_change(self) -> None:
+ """
+ Test that search with trivial RangeIndex doesn't add extra columns.
+ """
+ df_trivial = pd.DataFrame(
+ {"A": [1, 2, 3], "B": ["foo", "bars", "bare"]},
+ )
+ manager = self.factory.create()(df_trivial)
+ result = manager.search("bar")
+ result_df = nw.to_native(result.data)
+
+ # Should have no named index, no extra columns
+ # Note: filtering preserves original row indices (1, 2) not (0, 1)
+ expected = pd.DataFrame(
+ {"A": [2, 3], "B": ["bars", "bare"]},
+ index=pd.Index([1, 2]),
+ )
+ assert_frame_equal(result_df, expected)
+
+ def test_search_preserves_integer_index(self) -> None:
+ """
+ Test that search preserves integer index (non-RangeIndex).
+ This tests the case where index values are integers but not a RangeIndex.
+ """
+ df_int_index = pd.DataFrame(
+ {"value": ["foo", "bars", "bare"]},
+ index=pd.Index([100, 200, 300]),
+ )
+ manager = self.factory.create()(df_int_index)
+ result = manager.search("bar")
+ result_df = nw.to_native(result.data)
+
+ expected = pd.DataFrame(
+ {"value": ["bars", "bare"]},
+ index=pd.Index([200, 300]),
+ )
+ assert_frame_equal(result_df, expected)
+
+ def test_search_preserves_multiindex_with_integer_level_names(
+ self,
+ ) -> None:
+ """
+ Test that search works with MultiIndex that has integer level names.
+ This tests the bug where non-string column names cause set_index to fail.
+
+ When reset_index() creates columns from index levels, unnamed levels get
+ default names like 'level_0', 'level_1'. But if the index had non-string
+ names (e.g., integers), those become column names. The
+ _handle_non_string_column_names method converts these to strings,
+ but we need to use the converted names when restoring the index.
+ """
+ # Create a DataFrame with integer column names that will be used as index
+ df = pd.DataFrame(
+ {
+ 0: ["A", "A", "B", "B"],
+ 1: ["x", "y", "x", "y"],
+ "value": [1, 2, 3, 4],
+ }
+ )
+ df = df.set_index([0, 1])
+ manager = self.factory.create()(df)
+ result = manager.search("A")
+ result_df = nw.to_native(result.data)
+
+ expected = pd.DataFrame(
+ {"value": [1, 2]},
+ index=pd.MultiIndex.from_tuples(
+ [("A", "x"), ("A", "y")],
+ names=[0, 1],
+ ),
+ )
+ assert_frame_equal(result_df, expected)
+
def test_apply_formatting_does_not_modify_original_data(self) -> None:
original_data = self.data.copy()
format_mapping = {
@@ -1695,3 +1915,29 @@ class TestPandasTableManager(unittest.TestCase):
# Polygon geometry
assert isinstance(json_data[2]["geometry"], str)
assert "POLYGON" in json_data[2]["geometry"]
+
+ def test_search_with_index_column_name_conflict(self) -> None:
+ df = pd.DataFrame(
+ {"x": [10, 20, 30], "y": ["foo", "bar", "baz"]},
+ )
+ df.index = pd.Index([1, 2, 3], name="x")
+ manager = self.factory.create()(df)
+ # Should not raise ValueError: cannot insert x, already exists
+ result = manager.search("foo")
+ assert result.get_num_rows() == 1
+ # Index should be preserved
+ assert result._original_data.index.name == "x"
+
+ def test_search_with_multiindex_column_name_conflict(self) -> None:
+ df = pd.DataFrame(
+ {"x": [10, 20, 30], "y": ["foo", "bar", "baz"]},
+ index=pd.MultiIndex.from_tuples(
+ [(1, "a"), (2, "b"), (3, "c")], names=["x", "level"]
+ ),
+ )
+ manager = self.factory.create()(df)
+ # Should not raise ValueError: cannot insert x, already exists
+ result = manager.search("bar")
+ assert result.get_num_rows() == 1
+ # MultiIndex should be preserved with original names
+ assert list(result._original_data.index.names) == ["x", "level"]