Apache DataFusion SQL Query Engine
Account for TopK common-prefix converter memory (#26286)
kosiew5009f42e4a24
diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rsindex aee902f2b7..ef275b301a 100644--- a/datafusion/physical-plan/src/topk/mod.rs+++ b/datafusion/physical-plan/src/topk/mod.rs@@ -882,6 +882,10 @@ impl TopK {fn size(&self) -> usize {size_of::<Self>()+ self.row_converter.size()+ + self+ .common_sort_prefix_converter+ .as_ref()+ .map_or(0, RowConverter::size)+ self.scratch_rows.size()+ self.heap.size()}@@ -3306,6 +3310,47 @@ mod tests {)}+ #[test]+ fn test_topk_size_common_sort_prefix_converter() -> Result<()> {+ let schema = make_ab_schema();+ let sort_expr = PhysicalSortExpr {+ expr: col("a", schema.as_ref())?,+ options: SortOptions::default(),+ };+ let make_topk = |common_sort_prefix| {+ TopK::try_new(+ 0,+ Arc::clone(&schema),+ common_sort_prefix,+ LexOrdering::from([sort_expr.clone()]),+ 3,+ 2,+ Arc::new(RuntimeEnv::default()),+ &ExecutionPlanMetricsSet::new(),+ make_topk_filter(),+ )+ };+ let without_prefix = make_topk(vec![])?;+ let with_prefix = make_topk(vec![sort_expr.clone()])?;+ let converter_size = with_prefix+ .common_sort_prefix_converter+ .as_ref()+ .expect("common sort prefix converter")+ .size();+ assert!(converter_size > 0);+ assert!(without_prefix.common_sort_prefix_converter.is_none());+ assert_eq!(+ without_prefix.size(),+ size_of::<TopK>()+ + without_prefix.row_converter.size()+ + without_prefix.scratch_rows.size()+ + without_prefix.heap.size()+ );+ assert_eq!(with_prefix.size(), without_prefix.size() + converter_size);++ Ok(())+ }+/// Reads the `output_batches` and `output_rows` metrics recorded by an emit.fn output_batches_and_rows(metrics: &ExecutionPlanMetricsSet) -> (usize, usize) {let metrics = metrics.clone_inner();