Skip to content

Apache DataFusion SQL Query Engine

FilterExec should remap indices of parent dynamic filters (#20286)

Jack Kleeman0f74dbfda113
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index abd7b72fc9..13ba05f36e 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -32,7 +32,7 @@ use crate::common::can_project;
use crate::execution_plan::CardinalityEffect;
use crate::filter_pushdown::{
ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase,
- FilterPushdownPropagation, PushedDown, PushedDownPredicate,
+ FilterPushdownPropagation, PushedDown,
};
use crate::metrics::{MetricBuilder, MetricType};
use crate::projection::{
@@ -590,16 +590,9 @@ impl ExecutionPlan for FilterExec {
_config: &ConfigOptions,
) -> Result<FilterDescription> {
if !matches!(phase, FilterPushdownPhase::Pre) {
- // For non-pre phase, filters pass through unchanged
- let filter_supports = parent_filters
- .into_iter()
- .map(PushedDownPredicate::supported)
- .collect();
-
- return Ok(FilterDescription::new().with_child(ChildFilterDescription {
- parent_filters: filter_supports,
- self_filters: vec![],
- }));
+ let child =
+ ChildFilterDescription::from_child(&parent_filters, self.input())?;
+ return Ok(FilterDescription::new().with_child(child));
}
let child = ChildFilterDescription::from_child(&parent_filters, self.input())?
@@ -2011,4 +2004,57 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_filter_with_projection_remaps_post_phase_parent_filters() -> Result<()> {
+ // Test that FilterExec with a projection must remap parent dynamic
+ // filter column indices from its output schema to the input schema
+ // before passing them to the child.
+ let input_schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Utf8, false),
+ Field::new("c", DataType::Float64, false),
+ ]));
+ let input = Arc::new(EmptyExec::new(Arc::clone(&input_schema)));
+
+ // FilterExec: a > 0, projection=[c@2]
+ let predicate = Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("a", 0)),
+ Operator::Gt,
+ Arc::new(Literal::new(ScalarValue::Int32(Some(0)))),
+ ));
+ let filter = FilterExecBuilder::new(predicate, input)
+ .apply_projection(Some(vec![2]))?
+ .build()?;
+
+ // Output schema should be [c:Float64]
+ let output_schema = filter.schema();
+ assert_eq!(output_schema.fields().len(), 1);
+ assert_eq!(output_schema.field(0).name(), "c");
+
+ // Simulate a parent dynamic filter referencing output column c@0
+ let parent_filter: Arc<dyn PhysicalExpr> = Arc::new(Column::new("c", 0));
+
+ let config = ConfigOptions::new();
+ let desc = filter.gather_filters_for_pushdown(
+ FilterPushdownPhase::Post,
+ vec![parent_filter],
+ &config,
+ )?;
+
+ // The filter pushed to the child must reference c@2 (input schema),
+ // not c@0 (output schema).
+ let parent_filters = desc.parent_filters();
+ assert_eq!(parent_filters.len(), 1); // one child
+ assert_eq!(parent_filters[0].len(), 1); // one filter
+ let remapped = &parent_filters[0][0].predicate;
+ let display = format!("{remapped}");
+ assert_eq!(
+ display, "c@2",
+ "Post-phase parent filter column index must be remapped \
+ from output schema (c@0) to input schema (c@2)"
+ );
+
+ Ok(())
+ }
}