Apache DataFusion SQL Query Engine
chore: enforce `clippy::needless_pass_by_value` to `datafusion-physical-plan` (#18864)
Yongting Youc141fca12bfb
diff --git a/datafusion/physical-plan/src/aggregates/group_values/row.rs b/datafusion/physical-plan/src/aggregates/group_values/row.rsindex 34893fcc4e..d632a7f0ad 100644--- a/datafusion/physical-plan/src/aggregates/group_values/row.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/row.rs@@ -236,8 +236,7 @@ impl GroupValues for GroupValuesRows {// https://github.com/apache/datafusion/issues/7647for (field, array) in self.schema.fields.iter().zip(&mut output) {let expected = field.data_type();- *array =- dictionary_encode_if_necessary(Arc::<dyn Array>::clone(array), expected)?;+ *array = dictionary_encode_if_necessary(array, expected)?;}self.group_values = Some(group_values);@@ -259,7 +258,7 @@ impl GroupValues for GroupValuesRows {}fn dictionary_encode_if_necessary(- array: ArrayRef,+ array: &ArrayRef,expected: &DataType,) -> Result<ArrayRef> {match (expected, array.data_type()) {@@ -269,10 +268,7 @@ fn dictionary_encode_if_necessary(.iter().zip(struct_array.columns()).map(|(expected_field, column)| {- dictionary_encode_if_necessary(- Arc::<dyn Array>::clone(column),- expected_field.data_type(),- )+ dictionary_encode_if_necessary(column, expected_field.data_type())}).collect::<Result<Vec<_>>>()?;@@ -289,13 +285,13 @@ fn dictionary_encode_if_necessary(Arc::<arrow::datatypes::Field>::clone(expected_field),list.offsets().clone(),dictionary_encode_if_necessary(- Arc::<dyn Array>::clone(list.values()),+ list.values(),expected_field.data_type(),)?,list.nulls().cloned(),)?))}(DataType::Dictionary(_, _), _) => Ok(cast(array.as_ref(), expected)?),- (_, _) => Ok(Arc::<dyn Array>::clone(&array)),+ (_, _) => Ok(Arc::<dyn Array>::clone(array)),}}diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 5fafce0bea..6bf59fd3d3 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -626,7 +626,7 @@ impl AggregateExec {fn execute_typed(&self,partition: usize,- context: Arc<TaskContext>,+ context: &Arc<TaskContext>,) -> Result<StreamType> {// no group by at allif self.group_by.expr.is_empty() {@@ -761,7 +761,7 @@ impl AggregateExec {&self.input_order_mode}- fn statistics_inner(&self, child_statistics: Statistics) -> Result<Statistics> {+ fn statistics_inner(&self, child_statistics: &Statistics) -> Result<Statistics> {// TODO stats: group expressions:// - once expressions will be able to compute their own stats, use it here// - case where we group by on a column for which with have the `distinct` stat@@ -1020,7 +1020,7 @@ impl ExecutionPlan for AggregateExec {partition: usize,context: Arc<TaskContext>,) -> Result<SendableRecordBatchStream> {- self.execute_typed(partition, context)+ self.execute_typed(partition, &context).map(|stream| stream.into())}@@ -1033,7 +1033,8 @@ impl ExecutionPlan for AggregateExec {}fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {- self.statistics_inner(self.input().partition_statistics(partition)?)+ let child_statistics = self.input().partition_statistics(partition)?;+ self.statistics_inner(&child_statistics)}fn cardinality_effect(&self) -> CardinalityEffect {@@ -2220,7 +2221,7 @@ mod tests {Arc::clone(&input_schema),)?);- let stream = partial_aggregate.execute_typed(0, Arc::clone(&task_ctx))?;+ let stream = partial_aggregate.execute_typed(0, &task_ctx)?;// ensure that we really got the version we wantedmatch version {diff --git a/datafusion/physical-plan/src/aggregates/no_grouping.rs b/datafusion/physical-plan/src/aggregates/no_grouping.rsindex fc398427ac..a4b202f1ae 100644--- a/datafusion/physical-plan/src/aggregates/no_grouping.rs+++ b/datafusion/physical-plan/src/aggregates/no_grouping.rs@@ -68,14 +68,14 @@ impl AggregateStream {/// Create a new AggregateStreampub fn new(agg: &AggregateExec,- context: Arc<TaskContext>,+ context: &Arc<TaskContext>,partition: usize,) -> Result<Self> {let agg_schema = Arc::clone(&agg.schema);let agg_filter_expr = agg.filter_expr.clone();let baseline_metrics = BaselineMetrics::new(&agg.metrics, partition);- let input = agg.input.execute(partition, Arc::clone(&context))?;+ let input = agg.input.execute(partition, Arc::clone(context))?;let aggregate_expressions = aggregate_expressions(&agg.aggr_expr, &agg.mode, 0)?;let filter_expressions = match agg.mode {@@ -115,7 +115,7 @@ impl AggregateStream {let timer = elapsed_compute.timer();let result = aggregate_batch(&this.mode,- batch,+ &batch,&mut this.accumulators,&this.aggregate_expressions,&this.filter_expressions,@@ -195,7 +195,7 @@ impl RecordBatchStream for AggregateStream {/// TODO: Make this a member functionfn aggregate_batch(mode: &AggregateMode,- batch: RecordBatch,+ batch: &RecordBatch,accumulators: &mut [AccumulatorItem],expressions: &[Vec<Arc<dyn PhysicalExpr>>],filters: &[Option<Arc<dyn PhysicalExpr>>],@@ -215,8 +215,8 @@ fn aggregate_batch(.try_for_each(|((accum, expr), filter)| {// 1.2let batch = match filter {- Some(filter) => Cow::Owned(batch_filter(&batch, filter)?),- None => Cow::Borrowed(&batch),+ Some(filter) => Cow::Owned(batch_filter(batch, filter)?),+ None => Cow::Borrowed(batch),};// 1.3diff --git a/datafusion/physical-plan/src/aggregates/row_hash.rs b/datafusion/physical-plan/src/aggregates/row_hash.rsindex 0ca4abca2a..42f1fb016c 100644--- a/datafusion/physical-plan/src/aggregates/row_hash.rs+++ b/datafusion/physical-plan/src/aggregates/row_hash.rs@@ -445,7 +445,7 @@ impl GroupedHashAggregateStream {/// Create a new GroupedHashAggregateStreampub fn new(agg: &AggregateExec,- context: Arc<TaskContext>,+ context: &Arc<TaskContext>,partition: usize,) -> Result<Self> {debug!("Creating GroupedHashAggregateStream");@@ -454,7 +454,7 @@ impl GroupedHashAggregateStream {let agg_filter_expr = agg.filter_expr.clone();let batch_size = context.session_config().batch_size();- let input = agg.input.execute(partition, Arc::clone(&context))?;+ let input = agg.input.execute(partition, Arc::clone(context))?;let baseline_metrics = BaselineMetrics::new(&agg.metrics, partition);let group_by_metrics = GroupByMetrics::new(&agg.metrics, partition);@@ -685,7 +685,7 @@ impl Stream for GroupedHashAggregateStream {}// Do the grouping- self.group_aggregate_batch(batch)?;+ self.group_aggregate_batch(&batch)?;// If we can begin emitting rows, do so,// otherwise keep consuming input@@ -739,7 +739,7 @@ impl Stream for GroupedHashAggregateStream {self.spill_previous_if_necessary(&batch)?;// Do the grouping- self.group_aggregate_batch(batch)?;+ self.group_aggregate_batch(&batch)?;// If we can begin emitting rows, do so,// otherwise keep consuming input@@ -788,7 +788,7 @@ impl Stream for GroupedHashAggregateStream {if let Some(probe) = self.skip_aggregation_probe.as_mut() {probe.record_skipped(&batch);}- let states = self.transform_to_states(batch)?;+ let states = self.transform_to_states(&batch)?;return Poll::Ready(Some(Ok(states.record_output(&self.baseline_metrics))));@@ -881,12 +881,12 @@ impl RecordBatchStream for GroupedHashAggregateStream {impl GroupedHashAggregateStream {/// Perform group-by aggregation for the given [`RecordBatch`].- fn group_aggregate_batch(&mut self, batch: RecordBatch) -> Result<()> {+ fn group_aggregate_batch(&mut self, batch: &RecordBatch) -> Result<()> {// Evaluate the grouping expressionslet group_by_values = if self.spill_state.is_stream_merging {- evaluate_group_by(&self.spill_state.merging_group_by, &batch)?+ evaluate_group_by(&self.spill_state.merging_group_by, batch)?} else {- evaluate_group_by(&self.group_by, &batch)?+ evaluate_group_by(&self.group_by, batch)?};// Only create the timer if there are actual aggregate arguments to evaluate@@ -903,18 +903,18 @@ impl GroupedHashAggregateStream {// Evaluate the aggregation expressions.let input_values = if self.spill_state.is_stream_merging {- evaluate_many(&self.spill_state.merging_aggregate_arguments, &batch)?+ evaluate_many(&self.spill_state.merging_aggregate_arguments, batch)?} else {- evaluate_many(&self.aggregate_arguments, &batch)?+ evaluate_many(&self.aggregate_arguments, batch)?};drop(timer);// Evaluate the filter expressions, if any, against the inputslet filter_values = if self.spill_state.is_stream_merging {let filter_expressions = vec![None; self.accumulators.len()];- evaluate_optional(&filter_expressions, &batch)?+ evaluate_optional(&filter_expressions, batch)?} else {- evaluate_optional(&self.filter_expressions, &batch)?+ evaluate_optional(&self.filter_expressions, batch)?};for group_values in &group_by_values {@@ -1248,10 +1248,10 @@ impl GroupedHashAggregateStream {}/// Transforms input batch to intermediate aggregate state, without grouping it- fn transform_to_states(&self, batch: RecordBatch) -> Result<RecordBatch> {- let mut group_values = evaluate_group_by(&self.group_by, &batch)?;- let input_values = evaluate_many(&self.aggregate_arguments, &batch)?;- let filter_values = evaluate_optional(&self.filter_expressions, &batch)?;+ fn transform_to_states(&self, batch: &RecordBatch) -> Result<RecordBatch> {+ let mut group_values = evaluate_group_by(&self.group_by, batch)?;+ let input_values = evaluate_many(&self.aggregate_arguments, batch)?;+ let filter_values = evaluate_optional(&self.filter_expressions, batch)?;assert_eq_or_internal_err!(group_values.len(),diff --git a/datafusion/physical-plan/src/aggregates/topk/heap.rs b/datafusion/physical-plan/src/aggregates/topk/heap.rsindex 23ccf5e17e..83d76a919e 100644--- a/datafusion/physical-plan/src/aggregates/topk/heap.rs+++ b/datafusion/physical-plan/src/aggregates/topk/heap.rs@@ -326,13 +326,7 @@ impl<VAL: ValueType> TopKHeap<VAL> {}}- fn _tree_print(- &self,- idx: usize,- prefix: String,- is_tail: bool,- output: &mut String,- ) {+ fn _tree_print(&self, idx: usize, prefix: &str, is_tail: bool, output: &mut String) {if let Some(Some(hi)) = self.heap.get(idx) {let connector = if idx != 0 {if is_tail {@@ -357,10 +351,10 @@ impl<VAL: ValueType> TopKHeap<VAL> {let right_exists = right_idx < self.len;if left_exists {- self._tree_print(left_idx, child_prefix.clone(), !right_exists, output);+ self._tree_print(left_idx, &child_prefix, !right_exists, output);}if right_exists {- self._tree_print(right_idx, child_prefix, true, output);+ self._tree_print(right_idx, &child_prefix, true, output);}}}@@ -370,7 +364,7 @@ impl<VAL: ValueType> Display for TopKHeap<VAL> {fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {let mut output = String::new();if !self.heap.is_empty() {- self._tree_print(0, String::new(), true, &mut output);+ self._tree_print(0, "", true, &mut output);}write!(f, "{output}")}diff --git a/datafusion/physical-plan/src/aggregates/topk_stream.rs b/datafusion/physical-plan/src/aggregates/topk_stream.rsindex eb1b7543cb..c706b48e34 100644--- a/datafusion/physical-plan/src/aggregates/topk_stream.rs+++ b/datafusion/physical-plan/src/aggregates/topk_stream.rs@@ -54,13 +54,13 @@ pub struct GroupedTopKAggregateStream {impl GroupedTopKAggregateStream {pub fn new(aggr: &AggregateExec,- context: Arc<TaskContext>,+ context: &Arc<TaskContext>,partition: usize,limit: usize,) -> Result<Self> {let agg_schema = Arc::clone(&aggr.schema);let group_by = aggr.group_by.clone();- let input = aggr.input.execute(partition, Arc::clone(&context))?;+ let input = aggr.input.execute(partition, Arc::clone(context))?;let baseline_metrics = BaselineMetrics::new(&aggr.metrics, partition);let group_by_metrics = GroupByMetrics::new(&aggr.metrics, partition);let aggregate_arguments =@@ -97,11 +97,12 @@ impl RecordBatchStream for GroupedTopKAggregateStream {}impl GroupedTopKAggregateStream {- fn intern(&mut self, ids: ArrayRef, vals: ArrayRef) -> Result<()> {+ fn intern(&mut self, ids: &ArrayRef, vals: &ArrayRef) -> Result<()> {let _timer = self.group_by_metrics.time_calculating_group_ids.timer();let len = ids.len();- self.priority_map.set_batch(ids, Arc::clone(&vals));+ self.priority_map+ .set_batch(Arc::clone(ids), Arc::clone(vals));let has_nulls = vals.null_count() > 0;for row_idx in 0..len {@@ -167,7 +168,7 @@ impl Stream for GroupedTopKAggregateStream {let input_values = Arc::clone(&input_values[0][0]);// iterate over each column of group_by values- (*self).intern(group_by_values, input_values)?;+ (*self).intern(&group_by_values, &input_values)?;}// inner is done, emit all rows and switch to producing outputNone => {diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rsindex f0a3da7555..01f997f23d 100644--- a/datafusion/physical-plan/src/analyze.rs+++ b/datafusion/physical-plan/src/analyze.rs@@ -206,8 +206,8 @@ impl ExecutionPlan for AnalyzeExec {show_statistics,total_rows,duration,- captured_input,- captured_schema,+ &captured_input,+ &captured_schema,&metric_types,)};@@ -225,8 +225,8 @@ fn create_output_batch(show_statistics: bool,total_rows: usize,duration: std::time::Duration,- input: Arc<dyn ExecutionPlan>,- schema: SchemaRef,+ input: &Arc<dyn ExecutionPlan>,+ schema: &SchemaRef,metric_types: &[MetricType],) -> Result<RecordBatch> {let mut type_builder = StringBuilder::with_capacity(1, 1024);@@ -262,7 +262,7 @@ fn create_output_batch(}RecordBatch::try_new(- schema,+ Arc::clone(schema),vec![Arc::new(type_builder.finish()),Arc::new(plan_builder.finish()),diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rsindex 245342f5cd..553e3e26ce 100644--- a/datafusion/physical-plan/src/execution_plan.rs+++ b/datafusion/physical-plan/src/execution_plan.rs@@ -1118,6 +1118,7 @@ pub fn check_default_invariants<P: ExecutionPlan + ?Sized>(/// 1. RepartitionExec for changing the partition number between two `ExecutionPlan`s/// 2. CoalescePartitionsExec for collapsing all of the partitions into one without ordering guarantee/// 3. SortPreservingMergeExec for collapsing all of the sorted partitions into one with ordering guarantee+#[expect(clippy::needless_pass_by_value)]pub fn need_data_exchange(plan: Arc<dyn ExecutionPlan>) -> bool {plan.properties().evaluation_type == EvaluationType::Eager}@@ -1172,6 +1173,10 @@ pub async fn collect(////// Dropping the stream will abort the execution of the query, and free up/// any allocated resources+#[expect(+ clippy::needless_pass_by_value,+ reason = "Public API that historically takes owned Arcs"+)]pub fn execute_stream(plan: Arc<dyn ExecutionPlan>,context: Arc<TaskContext>,@@ -1236,6 +1241,10 @@ pub async fn collect_partitioned(////// Dropping the stream will abort the execution of the query, and free up/// any allocated resources+#[expect(+ clippy::needless_pass_by_value,+ reason = "Public API that historically takes owned Arcs"+)]pub fn execute_stream_partitioned(plan: Arc<dyn ExecutionPlan>,context: Arc<TaskContext>,@@ -1267,6 +1276,10 @@ pub fn execute_stream_partitioned(/// violate the `not null` constraints specified in the `sink_schema`. If there are/// such columns, it wraps the resulting stream to enforce the `not null` constraints/// by invoking the [`check_not_null_constraints`] function on each batch of the stream.+#[expect(+ clippy::needless_pass_by_value,+ reason = "Public API that historically takes owned Arcs"+)]pub fn execute_input_stream(input: Arc<dyn ExecutionPlan>,sink_schema: SchemaRef,diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex 8a05bfcee3..58185c8cdf 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -95,6 +95,7 @@ pub struct FilterExec {impl FilterExec {/// Create a FilterExec on an input+ #[expect(clippy::needless_pass_by_value)]pub fn try_new(predicate: Arc<dyn PhysicalExpr>,input: Arc<dyn ExecutionPlan>,@@ -204,12 +205,12 @@ impl FilterExec {/// Calculates `Statistics` for `FilterExec`, by applying selectivity (either default, or estimated) to input statistics.fn statistics_helper(- schema: SchemaRef,+ schema: &SchemaRef,input_stats: Statistics,predicate: &Arc<dyn PhysicalExpr>,default_selectivity: u8,) -> Result<Statistics> {- if !check_support(predicate, &schema) {+ if !check_support(predicate, schema) {let selectivity = default_selectivity as f64 / 100.0;let mut stats = input_stats.to_inexact();stats.num_rows = stats.num_rows.with_estimated_selectivity(selectivity);@@ -221,12 +222,10 @@ impl FilterExec {let num_rows = input_stats.num_rows;let total_byte_size = input_stats.total_byte_size;- let input_analysis_ctx = AnalysisContext::try_from_statistics(- &schema,- &input_stats.column_statistics,- )?;+ let input_analysis_ctx =+ AnalysisContext::try_from_statistics(schema, &input_stats.column_statistics)?;- let analysis_ctx = analyze(predicate, input_analysis_ctx, &schema)?;+ let analysis_ctx = analyze(predicate, input_analysis_ctx, schema)?;// Estimate (inexact) selectivity of predicatelet selectivity = analysis_ctx.selectivity.unwrap_or(1.0);@@ -283,8 +282,9 @@ impl FilterExec {) -> Result<PlanProperties> {// Combine the equal predicates with the input equivalence properties// to construct the equivalence properties:+ let schema = input.schema();let stats = Self::statistics_helper(- input.schema(),+ &schema,input.partition_statistics(None)?,predicate,default_selectivity,@@ -443,8 +443,9 @@ impl ExecutionPlan for FilterExec {fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {let input_stats = self.input.partition_statistics(partition)?;+ let schema = self.schema();let stats = Self::statistics_helper(- self.schema(),+ &schema,input_stats,self.predicate(),self.default_selectivity,diff --git a/datafusion/physical-plan/src/filter_pushdown.rs b/datafusion/physical-plan/src/filter_pushdown.rsindex f6b1b7448f..1274e954ea 100644--- a/datafusion/physical-plan/src/filter_pushdown.rs+++ b/datafusion/physical-plan/src/filter_pushdown.rs@@ -412,6 +412,7 @@ impl FilterDescription {/// This method automatically determines filter routing based on column analysis:/// - If all columns referenced by a filter exist in a child's schema, it can be pushed down/// - Otherwise, it cannot be pushed down to that child+ #[expect(clippy::needless_pass_by_value)]pub fn from_children(parent_filters: Vec<Arc<dyn PhysicalExpr>>,children: &[&Arc<dyn crate::ExecutionPlan>],diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 680e4368ae..03bf516ead 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -419,7 +419,7 @@ impl HashJoinExec {let cache = Self::compute_properties(&left,&right,- Arc::clone(&join_schema),+ &join_schema,*join_type,&on,partition_mode,@@ -550,7 +550,7 @@ impl HashJoinExec {fn compute_properties(left: &Arc<dyn ExecutionPlan>,right: &Arc<dyn ExecutionPlan>,- schema: SchemaRef,+ schema: &SchemaRef,join_type: JoinType,on: JoinOnRef,mode: PartitionMode,@@ -561,7 +561,7 @@ impl HashJoinExec {left.equivalence_properties().clone(),right.equivalence_properties().clone(),&join_type,- Arc::clone(&schema),+ Arc::clone(schema),&Self::maintains_input_order(join_type),Some(Self::probe_side()),on,@@ -605,9 +605,8 @@ impl HashJoinExec {// If contains projection, update the PlanProperties.if let Some(projection) = projection {// construct a map from the input expressions to the output expression of the Projection- let projection_mapping =- ProjectionMapping::from_indices(projection, &schema)?;- let out_schema = project_schema(&schema, Some(projection))?;+ let projection_mapping = ProjectionMapping::from_indices(projection, schema)?;+ let out_schema = project_schema(schema, Some(projection))?;output_partitioning =output_partitioning.project(&projection_mapping, &eq_properties);eq_properties = eq_properties.project(&projection_mapping, out_schema);@@ -852,7 +851,7 @@ impl ExecutionPlan for HashJoinExec {cache: Self::compute_properties(&children[0],&children[1],- Arc::clone(&self.join_schema),+ &self.join_schema,self.join_type,&self.on,self.mode,@@ -1045,7 +1044,7 @@ impl ExecutionPlan for HashJoinExec {let stats = estimate_join_statistics(self.left.partition_statistics(None)?,self.right.partition_statistics(None)?,- self.on.clone(),+ &self.on,&self.join_type,&self.join_schema,)?;@@ -1065,6 +1064,7 @@ impl ExecutionPlan for HashJoinExec {return Ok(None);}+ let schema = self.schema();if let Some(JoinData {projected_left_child,projected_right_child,@@ -1075,7 +1075,7 @@ impl ExecutionPlan for HashJoinExec {self.left(),self.right(),self.on(),- self.schema(),+ &schema,self.filter(),)? {Ok(Some(Arc::new(HashJoinExec::try_new(@@ -1299,10 +1299,8 @@ impl BuildSideState {bounds_accumulators: should_compute_dynamic_filters.then(|| {on_left- .iter()- .map(|expr| {- CollectLeftAccumulator::try_new(Arc::clone(expr), schema)- })+ .into_iter()+ .map(|expr| CollectLeftAccumulator::try_new(expr, schema)).collect::<Result<Vec<_>>>()}).transpose()?,diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rsindex 26533ce4c7..f16e2220df 100644--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs@@ -219,7 +219,7 @@ impl NestedLoopJoinExec {let cache = Self::compute_properties(&left,&right,- Arc::clone(&join_schema),+ &join_schema,*join_type,projection.as_ref(),)?;@@ -266,7 +266,7 @@ impl NestedLoopJoinExec {fn compute_properties(left: &Arc<dyn ExecutionPlan>,right: &Arc<dyn ExecutionPlan>,- schema: SchemaRef,+ schema: &SchemaRef,join_type: JoinType,projection: Option<&Vec<usize>>,) -> Result<PlanProperties> {@@ -275,7 +275,7 @@ impl NestedLoopJoinExec {left.equivalence_properties().clone(),right.equivalence_properties().clone(),&join_type,- Arc::clone(&schema),+ Arc::clone(schema),&Self::maintains_input_order(join_type),None,// No on columns in nested loop join@@ -310,9 +310,8 @@ impl NestedLoopJoinExec {if let Some(projection) = projection {// construct a map from the input expressions to the output expression of the Projection- let projection_mapping =- ProjectionMapping::from_indices(projection, &schema)?;- let out_schema = project_schema(&schema, Some(projection))?;+ let projection_mapping = ProjectionMapping::from_indices(projection, schema)?;+ let out_schema = project_schema(schema, Some(projection))?;output_partitioning =output_partitioning.project(&projection_mapping, &eq_properties);eq_properties = eq_properties.project(&projection_mapping, out_schema);@@ -555,10 +554,11 @@ impl ExecutionPlan for NestedLoopJoinExec {if partition.is_some() {return Ok(Statistics::new_unknown(&self.schema()));}+ let join_columns = Vec::new();estimate_join_statistics(self.left.partition_statistics(None)?,self.right.partition_statistics(None)?,- vec![],+ &join_columns,&self.join_type,&self.schema(),)@@ -576,6 +576,7 @@ impl ExecutionPlan for NestedLoopJoinExec {return Ok(None);}+ let schema = self.schema();if let Some(JoinData {projected_left_child,projected_right_child,@@ -586,7 +587,7 @@ impl ExecutionPlan for NestedLoopJoinExec {self.left(),self.right(),&[],- self.schema(),+ &schema,self.filter(),)? {Ok(Some(Arc::new(NestedLoopJoinExec::try_new(@@ -1656,11 +1657,12 @@ impl NestedLoopJoinStream {}let bitmap_sliced = BooleanArray::new(bitmap_sliced.finish(), None);+ let right_schema = self.right_data.schema();build_unmatched_batch(- Arc::clone(&self.output_schema),+ &self.output_schema,&left_batch_sliced,bitmap_sliced,- self.right_data.schema(),+ &right_schema,&self.column_indices,self.join_type,JoinSide::Left,@@ -1683,10 +1685,10 @@ impl NestedLoopJoinStream {let left_schema = left_data.batch().schema();let res = build_unmatched_batch(- Arc::clone(&self.output_schema),+ &self.output_schema,&cur_right_batch,right_batch_bitmap,- left_schema,+ &left_schema,&self.column_indices,self.join_type,JoinSide::Right,@@ -1973,7 +1975,7 @@ fn build_row_join_batch(/// If Some, that's the result batch/// If None, it's not for this special case. Continue execution.fn build_unmatched_batch_empty_schema(- output_schema: SchemaRef,+ output_schema: &SchemaRef,batch_bitmap: &BooleanArray,// For left/right/full joins, it needs to fill nulls for another sidejoin_type: JoinType,@@ -1991,7 +1993,7 @@ fn build_unmatched_batch_empty_schema(if output_schema.fields().is_empty() {Ok(Some(create_record_batch_with_empty_schema(- Arc::clone(&output_schema),+ Arc::clone(output_schema),result_size,)?))} else {@@ -2051,11 +2053,11 @@ fn create_record_batch_with_empty_schema(/// Null(bool) Null(Int32) 1/// Null(bool) Null(Int32) 3fn build_unmatched_batch(- output_schema: SchemaRef,+ output_schema: &SchemaRef,batch: &RecordBatch,batch_bitmap: BooleanArray,// For left/right/full joins, it needs to fill nulls for another side- another_side_schema: SchemaRef,+ another_side_schema: &SchemaRef,col_indices: &[ColumnIndex],join_type: JoinType,batch_side: JoinSide,@@ -2065,11 +2067,9 @@ fn build_unmatched_batch(debug_assert_ne!(batch_side, JoinSide::None);// Handle special case (see function comment)- if let Some(batch) = build_unmatched_batch_empty_schema(- Arc::clone(&output_schema),- &batch_bitmap,- join_type,- )? {+ if let Some(batch) =+ build_unmatched_batch_empty_schema(output_schema, &batch_bitmap, join_type)?+ {return Ok(Some(batch));}@@ -2116,7 +2116,7 @@ fn build_unmatched_batch(debug_assert_ne!(batch_side, JoinSide::None);let opposite_side = batch_side.negate();- build_row_join_batch(&output_schema, &left_null_batch, 0, batch, Some(flipped_bitmap), col_indices, opposite_side)+ build_row_join_batch(output_schema, &left_null_batch, 0, batch, Some(flipped_bitmap), col_indices, opposite_side)},JoinType::RightSemi | JoinType::RightAnti | JoinType::LeftSemi | JoinType::LeftAnti => {@@ -2149,7 +2149,7 @@ fn build_unmatched_batch(columns.push(filtered_col);}- Ok(Some(RecordBatch::try_new(Arc::clone(&output_schema), columns)?))+ Ok(Some(RecordBatch::try_new(Arc::clone(output_schema), columns)?))},JoinType::RightMark | JoinType::LeftMark => {if join_type == JoinType::RightMark {@@ -2181,7 +2181,7 @@ fn build_unmatched_batch(}}- Ok(Some(RecordBatch::try_new(Arc::clone(&output_schema), columns)?))+ Ok(Some(RecordBatch::try_new(Arc::clone(output_schema), columns)?))}_ => internal_err!("If batch is at right side, this function must be handling Full/Right/RightSemi/RightAnti/RightMark joins"),}@@ -2246,7 +2246,8 @@ pub(crate) mod tests {source = source.try_with_sort_information(vec![ordering]).unwrap();}- Arc::new(TestMemoryExec::update_cache(Arc::new(source)))+ let source = Arc::new(source);+ Arc::new(TestMemoryExec::update_cache(&source))}fn build_left_table() -> Arc<dyn ExecutionPlan> {diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rsindex 646905e0d7..695e73f109 100644--- a/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs+++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs@@ -280,7 +280,7 @@ impl ClassicPWMJStream {let batch = resolve_classic_join(buffered_side,stream_batch,- Arc::clone(&self.schema),+ &self.schema,self.operator,self.sort_option,self.join_type,@@ -455,7 +455,7 @@ impl Stream for ClassicPWMJStream {fn resolve_classic_join(buffered_side: &mut BufferedSideReadyState,stream_batch: &SortedStreamBatch,- join_schema: Arc<Schema>,+ join_schema: &SchemaRef,operator: Operator,sort_options: SortOptions,join_type: JoinType,@@ -504,7 +504,7 @@ fn resolve_classic_join(buffered_side,stream_batch,join_type,- Arc::clone(&join_schema),+ join_schema,)?;batch_process_state.output_batches.push_batch(batch)?;@@ -532,7 +532,7 @@ fn resolve_classic_join(buffered_side,stream_batch,join_type,- Arc::clone(&join_schema),+ join_schema,)?;// Flush batch and update pointers if we have a completed batch@@ -579,14 +579,14 @@ fn resolve_classic_join(let batch = create_unmatched_batch(&mut batch_process_state.unmatched_indices,stream_batch,- Arc::clone(&join_schema),+ join_schema,)?;batch_process_state.output_batches.push_batch(batch)?;}batch_process_state.continue_process = false;- Ok(RecordBatch::new_empty(Arc::clone(&join_schema)))+ Ok(RecordBatch::new_empty(Arc::clone(join_schema)))}// Builds a record batch from indices ranges on the buffered and streamed side.@@ -599,7 +599,7 @@ fn build_matched_indices_and_set_buffered_bitmap(buffered_side: &mut BufferedSideReadyState,stream_batch: &SortedStreamBatch,join_type: JoinType,- join_schema: Arc<Schema>,+ join_schema: &SchemaRef,) -> Result<RecordBatch> {// Mark the buffered indices as visitedif need_produce_result_in_final(join_type) {@@ -622,7 +622,7 @@ fn build_matched_indices_and_set_buffered_bitmap(buffered_columns.extend(streamed_columns);Ok(RecordBatch::try_new(- Arc::clone(&join_schema),+ Arc::clone(join_schema),buffered_columns,)?)}@@ -631,7 +631,7 @@ fn build_matched_indices_and_set_buffered_bitmap(fn create_unmatched_batch(streamed_indices: &mut PrimitiveBuilder<UInt32Type>,stream_batch: &SortedStreamBatch,- join_schema: Arc<Schema>,+ join_schema: &SchemaRef,) -> Result<RecordBatch> {let streamed_indices = streamed_indices.finish();let new_stream_batch = take_record_batch(&stream_batch.batch, &streamed_indices)?;@@ -649,7 +649,7 @@ fn create_unmatched_batch(buffered_columns.extend(streamed_columns);Ok(RecordBatch::try_new(- Arc::clone(&join_schema),+ Arc::clone(join_schema),buffered_columns,)?)}diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rsindex bc139f43b7..daf47603c2 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs@@ -532,7 +532,7 @@ impl ExecutionPlan for SortMergeJoinExec {estimate_join_statistics(self.left.partition_statistics(None)?,self.right.partition_statistics(None)?,- self.on.clone(),+ &self.on,&self.join_type,&self.schema,)diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/stream.rsindex 0325e37d42..9acaba1c89 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs@@ -857,7 +857,7 @@ impl SortMergeJoinStream {}}- fn free_reservation(&mut self, buffered_batch: BufferedBatch) -> Result<()> {+ fn free_reservation(&mut self, buffered_batch: &BufferedBatch) -> Result<()> {// Shrink memory usage for in-memory batches onlyif let BufferedBatchState::InMemory(_) = buffered_batch.batch {self.reservation@@ -915,7 +915,7 @@ impl SortMergeJoinStream {self.buffered_data.batches.pop_front(){self.produce_buffered_not_matched(&mut buffered_batch)?;- self.free_reservation(buffered_batch)?;+ self.free_reservation(&buffered_batch)?;}} else {// If the head batch is not fully processed, break the loop.@@ -1533,16 +1533,16 @@ impl SortMergeJoinStream {&out_mask};- self.filter_record_batch_by_join_type(record_batch, corrected_mask)+ self.filter_record_batch_by_join_type(&record_batch, corrected_mask)}fn filter_record_batch_by_join_type(&mut self,- record_batch: RecordBatch,+ record_batch: &RecordBatch,corrected_mask: &BooleanArray,) -> Result<RecordBatch> {let mut filtered_record_batch =- filter_record_batch(&record_batch, corrected_mask)?;+ filter_record_batch(record_batch, corrected_mask)?;let left_columns_length = self.streamed_schema.fields.len();let right_columns_length = self.buffered_schema.fields.len();@@ -1551,7 +1551,7 @@ impl SortMergeJoinStream {JoinType::Left | JoinType::LeftMark | JoinType::Right | JoinType::RightMark) {let null_mask = compute::not(corrected_mask)?;- let null_joined_batch = filter_record_batch(&record_batch, &null_mask)?;+ let null_joined_batch = filter_record_batch(record_batch, &null_mask)?;let mut right_columns = create_unmatched_columns(self.join_type,@@ -1609,7 +1609,7 @@ impl SortMergeJoinStream {// Find rows which joined by key but Filter predicate evaluated as falselet joined_filter_not_matched_mask = compute::not(corrected_mask)?;let joined_filter_not_matched_batch =- filter_record_batch(&record_batch, &joined_filter_not_matched_mask)?;+ filter_record_batch(record_batch, &joined_filter_not_matched_mask)?;// Add left unmatched rows adding the right side as nullslet right_null_columns = selfdiff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rsindex 8b1d1d2e11..8b3677713a 100644--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs@@ -1399,7 +1399,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {return Poll::Ready(Ok(StatefulStreamResult::Continue));}self.set_state(SHJStreamState::PullLeft);- Poll::Ready(self.process_batch_from_right(batch))+ Poll::Ready(self.process_batch_from_right(&batch))}Some(Err(e)) => Poll::Ready(Err(e)),None => {@@ -1428,7 +1428,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {return Poll::Ready(Ok(StatefulStreamResult::Continue));}self.set_state(SHJStreamState::PullRight);- Poll::Ready(self.process_batch_from_left(batch))+ Poll::Ready(self.process_batch_from_left(&batch))}Some(Err(e)) => Poll::Ready(Err(e)),None => {@@ -1457,7 +1457,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {if batch.num_rows() == 0 {return Poll::Ready(Ok(StatefulStreamResult::Continue));}- Poll::Ready(self.process_batch_after_right_end(batch))+ Poll::Ready(self.process_batch_after_right_end(&batch))}Some(Err(e)) => Poll::Ready(Err(e)),None => {@@ -1488,7 +1488,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {if batch.num_rows() == 0 {return Poll::Ready(Ok(StatefulStreamResult::Continue));}- Poll::Ready(self.process_batch_after_left_end(batch))+ Poll::Ready(self.process_batch_after_left_end(&batch))}Some(Err(e)) => Poll::Ready(Err(e)),None => {@@ -1518,7 +1518,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {fn process_batch_from_right(&mut self,- batch: RecordBatch,+ batch: &RecordBatch,) -> Result<StatefulStreamResult<Option<RecordBatch>>> {self.perform_join_for_given_side(batch, JoinSide::Right).map(|maybe_batch| {@@ -1532,7 +1532,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {fn process_batch_from_left(&mut self,- batch: RecordBatch,+ batch: &RecordBatch,) -> Result<StatefulStreamResult<Option<RecordBatch>>> {self.perform_join_for_given_side(batch, JoinSide::Left).map(|maybe_batch| {@@ -1546,14 +1546,14 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {fn process_batch_after_left_end(&mut self,- right_batch: RecordBatch,+ right_batch: &RecordBatch,) -> Result<StatefulStreamResult<Option<RecordBatch>>> {self.process_batch_from_right(right_batch)}fn process_batch_after_right_end(&mut self,- left_batch: RecordBatch,+ left_batch: &RecordBatch,) -> Result<StatefulStreamResult<Option<RecordBatch>>> {self.process_batch_from_left(left_batch)}@@ -1632,7 +1632,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {/// 5. Combines the results and returns a combined batch or `None` if no batch was produced.fn perform_join_for_given_side(&mut self,- probe_batch: RecordBatch,+ probe_batch: &RecordBatch,probe_side: JoinSide,) -> Result<Option<RecordBatch>> {let (@@ -1662,7 +1662,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {probe_side_metrics.input_batches.add(1);probe_side_metrics.input_rows.add(probe_batch.num_rows());// Update the internal state of the hash joiner for the build side:- probe_hash_joiner.update_internal_state(&probe_batch, &self.random_state)?;+ probe_hash_joiner.update_internal_state(probe_batch, &self.random_state)?;// Join the two sides:let equal_result = join_with_probe_batch(build_hash_joiner,@@ -1670,7 +1670,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {&self.schema,self.join_type,self.filter.as_ref(),- &probe_batch,+ probe_batch,&self.column_indices,&self.random_state,self.null_equality,@@ -1691,7 +1691,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {calculate_filter_expr_intervals(&build_hash_joiner.input_buffer,build_side_sorted_filter_expr,- &probe_batch,+ probe_batch,probe_side_sorted_filter_expr,)?;let prune_length = build_hash_joinerdiff --git a/datafusion/physical-plan/src/joins/test_utils.rs b/datafusion/physical-plan/src/joins/test_utils.rsindex de288724c4..58338bd860 100644--- a/datafusion/physical-plan/src/joins/test_utils.rs+++ b/datafusion/physical-plan/src/joins/test_utils.rs@@ -534,9 +534,11 @@ pub fn create_memory_table(let right_schema = right_partition[0].schema();let right = TestMemoryExec::try_new(&[right_partition], right_schema, None)?.try_with_sort_information(right_sorted)?;+ let left = Arc::new(left);+ let right = Arc::new(right);Ok((- Arc::new(TestMemoryExec::update_cache(Arc::new(left))),- Arc::new(TestMemoryExec::update_cache(Arc::new(right))),+ Arc::new(TestMemoryExec::update_cache(&left)),+ Arc::new(TestMemoryExec::update_cache(&right)),))}diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rsindex 4cba85a851..9087ac415f 100644--- a/datafusion/physical-plan/src/joins/utils.rs+++ b/datafusion/physical-plan/src/joins/utils.rs@@ -414,11 +414,11 @@ struct PartialJoinStatistics {pub(crate) fn estimate_join_statistics(left_stats: Statistics,right_stats: Statistics,- on: JoinOn,+ on: &JoinOn,join_type: &JoinType,schema: &Schema,) -> Result<Statistics> {- let join_stats = estimate_join_cardinality(join_type, left_stats, right_stats, &on);+ let join_stats = estimate_join_cardinality(join_type, left_stats, right_stats, on);let (num_rows, column_statistics) = match join_stats {Some(stats) => (Precision::Inexact(stats.num_rows), stats.column_statistics),None => (Precision::Absent, Statistics::unknown_column(schema)),@@ -567,16 +567,26 @@ fn estimate_inner_join_cardinality(return Some(estimation);};+ let Statistics {+ num_rows: left_num_rows,+ column_statistics: left_column_statistics,+ ..+ } = left_stats;+ let Statistics {+ num_rows: right_num_rows,+ column_statistics: right_column_statistics,+ ..+ } = right_stats;+// The algorithm here is partly based on the non-histogram selectivity estimation// from Spark's Catalyst optimizer.let mut join_selectivity = Precision::Absent;- for (left_stat, right_stat) in left_stats- .column_statistics+ for (left_stat, right_stat) in left_column_statistics.iter()- .zip(right_stats.column_statistics.iter())+ .zip(right_column_statistics.iter()){- let left_max_distinct = max_distinct_count(&left_stats.num_rows, left_stat);- let right_max_distinct = max_distinct_count(&right_stats.num_rows, right_stat);+ let left_max_distinct = max_distinct_count(&left_num_rows, left_stat);+ let right_max_distinct = max_distinct_count(&right_num_rows, right_stat);let max_distinct = left_max_distinct.max(&right_max_distinct);if max_distinct.get_value().is_some() {// Seems like there are a few implementations of this algorithm that implement@@ -1131,8 +1141,8 @@ pub(crate) fn append_right_indices() -> Result<(UInt64Array, UInt32Array)> {if preserve_order_for_right {Ok(append_probe_indices_in_order(- left_indices,- right_indices,+ &left_indices,+ &right_indices,adjust_range,))} else {@@ -1276,8 +1286,8 @@ fn build_range_bitmap<T: ArrowPrimitiveType>(/// - A `PrimitiveArray` of `UInt64Type` with the newly constructed build indices./// - A `PrimitiveArray` of `UInt32Type` with the newly constructed probe indices.fn append_probe_indices_in_order(- build_indices: PrimitiveArray<UInt64Type>,- probe_indices: PrimitiveArray<UInt32Type>,+ build_indices: &PrimitiveArray<UInt64Type>,+ probe_indices: &PrimitiveArray<UInt32Type>,range: Range<usize>,) -> (PrimitiveArray<UInt64Type>, PrimitiveArray<UInt32Type>) {// Builders for new indices:diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rsindex 17628fd8ad..b74baf2d06 100644--- a/datafusion/physical-plan/src/lib.rs+++ b/datafusion/physical-plan/src/lib.rs@@ -23,6 +23,9 @@// Make sure fast / cheap clones on Arc are explicit:// https://github.com/apache/datafusion/issues/11143#![deny(clippy::clone_on_ref_ptr)]+// https://github.com/apache/datafusion/issues/18503+#![deny(clippy::needless_pass_by_value)]+#![cfg_attr(test, allow(clippy::needless_pass_by_value))]//! Traits for physical query plan, supporting parallel execution for partitioned relations.//!diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex abd8daa3fd..0b8c4ee5fb 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -480,7 +480,7 @@ pub fn try_pushdown_through_join(join_left: &Arc<dyn ExecutionPlan>,join_right: &Arc<dyn ExecutionPlan>,join_on: JoinOnRef,- schema: SchemaRef,+ schema: &SchemaRef,filter: Option<&JoinFilter>,) -> Result<Option<JoinData>> {// Convert projected expressions to columns. We can not proceed if this is not possible.@@ -493,7 +493,7 @@ pub fn try_pushdown_through_join(if !join_allows_pushdown(&projection_as_columns,- &schema,+ schema,far_right_left_col_ind,far_left_right_col_ind,) {diff --git a/datafusion/physical-plan/src/recursive_query.rs b/datafusion/physical-plan/src/recursive_query.rsindex 2c3965c0b6..7b966ed23d 100644--- a/datafusion/physical-plan/src/recursive_query.rs+++ b/datafusion/physical-plan/src/recursive_query.rs@@ -83,7 +83,7 @@ impl RecursiveQueryExec {// Each recursive query needs its own work tablelet work_table = Arc::new(WorkTable::new());// Use the same work table for both the WorkTableExec and the recursive term- let recursive_term = assign_work_table(recursive_term, Arc::clone(&work_table))?;+ let recursive_term = assign_work_table(recursive_term, &work_table)?;let cache = Self::compute_properties(static_term.schema());Ok(RecursiveQueryExec {name,@@ -346,12 +346,12 @@ impl RecursiveQueryStream {fn assign_work_table(plan: Arc<dyn ExecutionPlan>,- work_table: Arc<WorkTable>,+ work_table: &Arc<WorkTable>,) -> Result<Arc<dyn ExecutionPlan>> {let mut work_table_refs = 0;plan.transform_down(|plan| {if let Some(new_plan) =- plan.with_new_state(Arc::clone(&work_table) as Arc<dyn Any + Send + Sync>)+ plan.with_new_state(Arc::clone(work_table) as Arc<dyn Any + Send + Sync>){if work_table_refs > 0 {not_impl_err!(diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 6249a0e9b8..9dde0fc96e 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -228,10 +228,10 @@ impl Debug for RepartitionExecState {impl RepartitionExecState {fn ensure_input_streams_initialized(&mut self,- input: Arc<dyn ExecutionPlan>,- metrics: ExecutionPlanMetricsSet,+ input: &Arc<dyn ExecutionPlan>,+ metrics: &ExecutionPlanMetricsSet,output_partitions: usize,- ctx: Arc<TaskContext>,+ ctx: &Arc<TaskContext>,) -> Result<()> {if !matches!(self, RepartitionExecState::NotInitialized) {return Ok(());@@ -241,10 +241,10 @@ impl RepartitionExecState {let mut streams_and_metrics = Vec::with_capacity(num_input_partitions);for i in 0..num_input_partitions {- let metrics = RepartitionMetrics::new(i, output_partitions, &metrics);+ let metrics = RepartitionMetrics::new(i, output_partitions, metrics);let timer = metrics.fetch_time.timer();- let stream = input.execute(i, Arc::clone(&ctx))?;+ let stream = input.execute(i, Arc::clone(ctx))?;timer.done();streams_and_metrics.push((stream, metrics));@@ -256,21 +256,21 @@ impl RepartitionExecState {#[expect(clippy::too_many_arguments)]fn consume_input_streams(&mut self,- input: Arc<dyn ExecutionPlan>,- metrics: ExecutionPlanMetricsSet,- partitioning: Partitioning,+ input: &Arc<dyn ExecutionPlan>,+ metrics: &ExecutionPlanMetricsSet,+ partitioning: &Partitioning,preserve_order: bool,- name: String,- context: Arc<TaskContext>,+ name: &str,+ context: &Arc<TaskContext>,spill_manager: SpillManager,) -> Result<&mut ConsumingInputStreamsState> {let streams_and_metrics = match self {RepartitionExecState::NotInitialized => {self.ensure_input_streams_initialized(- Arc::clone(&input),- metrics.clone(),+ input,+ metrics,partitioning.partition_count(),- Arc::clone(&context),+ context,)?;let RepartitionExecState::InputStreamsInitialized(value) = self else {// This cannot happen, as ensure_input_streams_initialized() was just called,@@ -854,10 +854,10 @@ impl ExecutionPlan for RepartitionExec {let state = Arc::clone(&self.state);if let Some(mut state) = state.try_lock() {state.ensure_input_streams_initialized(- Arc::clone(&input),- metrics.clone(),+ &input,+ &metrics,partitioning.partition_count(),- Arc::clone(&context),+ &context,)?;}@@ -869,12 +869,12 @@ impl ExecutionPlan for RepartitionExec {// lock mutexeslet mut state = state.lock();let state = state.consume_input_streams(- Arc::clone(&input),- metrics.clone(),- partitioning,+ &input,+ &metrics,+ &partitioning,preserve_order,- name.clone(),- Arc::clone(&context),+ &name,+ &context,spill_manager.clone(),)?;@@ -1106,8 +1106,7 @@ impl RepartitionExec {partitioning: Partitioning,) -> Result<Self> {let preserve_order = false;- let cache =- Self::compute_properties(&input, partitioning.clone(), preserve_order);+ let cache = Self::compute_properties(&input, partitioning, preserve_order);Ok(RepartitionExec {input,state: Default::default(),@@ -2495,7 +2494,8 @@ mod test {// Create physical plan with order preservationlet exec = TestMemoryExec::try_new(&input_partitions, Arc::clone(&schema), None)?.try_with_sort_information(vec![sort_exprs.clone(), sort_exprs])?;- let exec = Arc::new(TestMemoryExec::update_cache(Arc::new(exec)));+ let exec = Arc::new(exec);+ let exec = Arc::new(TestMemoryExec::update_cache(&exec));// Repartition into 3 partitions with order preservation// We expect 1 batch per output partition after repartitioninglet exec = RepartitionExec::try_new(exec, Partitioning::RoundRobinBatch(3))?@@ -2611,7 +2611,8 @@ mod test {// Create physical plan with hash partitioninglet exec = TestMemoryExec::try_new(&input_partitions, Arc::clone(&schema), None)?;- let exec = Arc::new(TestMemoryExec::update_cache(Arc::new(exec)));+ let exec = Arc::new(exec);+ let exec = Arc::new(TestMemoryExec::update_cache(&exec));// Hash partition into 2 partitions by column c0let hash_expr = col("c0", &schema)?;let exec =@@ -2695,11 +2696,11 @@ mod test {schema: &SchemaRef,sort_exprs: LexOrdering,) -> Arc<dyn ExecutionPlan> {- Arc::new(TestMemoryExec::update_cache(Arc::new(- TestMemoryExec::try_new(&[vec![]], Arc::clone(schema), None)- .unwrap()- .try_with_sort_information(vec![sort_exprs])- .unwrap(),- )))+ let exec = TestMemoryExec::try_new(&[vec![]], Arc::clone(schema), None)+ .unwrap()+ .try_with_sort_information(vec![sort_exprs])+ .unwrap();+ let exec = Arc::new(exec);+ Arc::new(TestMemoryExec::update_cache(&exec))}}diff --git a/datafusion/physical-plan/src/sorts/stream.rs b/datafusion/physical-plan/src/sorts/stream.rsindex f419247c82..b0c631cf91 100644--- a/datafusion/physical-plan/src/sorts/stream.rs+++ b/datafusion/physical-plan/src/sorts/stream.rs@@ -99,8 +99,8 @@ impl ReusableRows {})}// save the Rows- fn save(&mut self, stream_idx: usize, rows: Arc<Rows>) {- self.inner[stream_idx][1] = Some(Arc::clone(&rows));+ fn save(&mut self, stream_idx: usize, rows: &Arc<Rows>) {+ self.inner[stream_idx][1] = Some(Arc::clone(rows));// swap the current with the previous one, so that the next poll can reuse the Rows from the previous polllet [a, b] = &mut self.inner[stream_idx];std::mem::swap(a, b);@@ -177,7 +177,7 @@ impl RowCursorStream {let rows = Arc::new(rows);- self.rows.save(stream_idx, Arc::clone(&rows));+ self.rows.save(stream_idx, &rows);// track the memory in the newly created Rows.let mut rows_reservation = self.reservation.new_empty();diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rsindex 58fd016a63..6be7edcf32 100644--- a/datafusion/physical-plan/src/spill/mod.rs+++ b/datafusion/physical-plan/src/spill/mod.rs@@ -227,6 +227,7 @@ impl RecordBatchStream for SpillReaderStream {since = "46.0.0",note = "This method is deprecated. Use `SpillManager::spill_record_batch_by_size` instead.")]+#[expect(clippy::needless_pass_by_value)]pub fn spill_record_batch_by_size(batch: &RecordBatch,path: PathBuf,diff --git a/datafusion/physical-plan/src/test.rs b/datafusion/physical-plan/src/test.rsindex bc32192917..4f7b843262 100644--- a/datafusion/physical-plan/src/test.rs+++ b/datafusion/physical-plan/src/test.rs@@ -271,10 +271,9 @@ impl TestMemoryExec {}// Equivalent of `DataSourceExec::new`- pub fn update_cache(source: Arc<TestMemoryExec>) -> TestMemoryExec {+ pub fn update_cache(source: &Arc<TestMemoryExec>) -> TestMemoryExec {let cache = source.compute_properties();- let source = &*source;- let mut source = source.clone();+ let mut source = (**source).clone();source.cache = cache;source}diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rsindex 96356e50d7..99af9b8f7c 100644--- a/datafusion/physical-plan/src/topk/mod.rs+++ b/datafusion/physical-plan/src/topk/mod.rs@@ -178,7 +178,8 @@ impl TopK {/// Create a new [`TopK`] that stores the top `k` values, as/// defined by the sort expressions in `expr`.// TODO: make a builder or some other nicer API- #[allow(clippy::too_many_arguments)]+ #[expect(clippy::too_many_arguments)]+ #[expect(clippy::needless_pass_by_value)]pub fn try_new(partition_id: usize,schema: SchemaRef,@@ -226,6 +227,7 @@ impl TopK {/// Insert `batch`, remembering if any of its values are among/// the top k seen so far.+ #[expect(clippy::needless_pass_by_value)]pub fn insert_batch(&mut self, batch: RecordBatch) -> Result<()> {// Updates on droplet baseline = self.metrics.baseline.clone();@@ -377,7 +379,7 @@ impl TopK {};// Build the filter expression OUTSIDE any synchronization- let predicate = Self::build_filter_expression(&self.expr, thresholds)?;+ let predicate = Self::build_filter_expression(&self.expr, &thresholds)?;let new_threshold = new_threshold_row.to_vec();// update the threshold. Since there was a lock gap, we must check if it is still the best@@ -420,7 +422,7 @@ impl TopK {/// This is now called outside of any locks to reduce critical section time.fn build_filter_expression(sort_exprs: &[PhysicalSortExpr],- thresholds: Vec<ScalarValue>,+ thresholds: &[ScalarValue],) -> Result<Option<Arc<dyn PhysicalExpr>>> {// Create filter expressions for each thresholdlet mut filters: Vec<Arc<dyn PhysicalExpr>> =diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rsindex 1432c2b09b..1f4bc8817e 100644--- a/datafusion/physical-plan/src/union.rs+++ b/datafusion/physical-plan/src/union.rs@@ -699,7 +699,7 @@ impl Stream for CombinedRecordBatchStream {fn col_stats_union(mut left: ColumnStatistics,- right: ColumnStatistics,+ right: &ColumnStatistics,) -> ColumnStatistics {left.distinct_count = Precision::Absent;left.min_value = left.min_value.min(&right.min_value);@@ -711,12 +711,18 @@ fn col_stats_union(}fn stats_union(mut left: Statistics, right: Statistics) -> Statistics {- left.num_rows = left.num_rows.add(&right.num_rows);- left.total_byte_size = left.total_byte_size.add(&right.total_byte_size);+ let Statistics {+ num_rows: right_num_rows,+ total_byte_size: right_total_bytes,+ column_statistics: right_column_statistics,+ ..+ } = right;+ left.num_rows = left.num_rows.add(&right_num_rows);+ left.total_byte_size = left.total_byte_size.add(&right_total_bytes);left.column_statistics = left.column_statistics.into_iter()- .zip(right.column_statistics)+ .zip(right_column_statistics.iter()).map(|(a, b)| col_stats_union(a, b)).collect::<Vec<_>>();left@@ -926,14 +932,14 @@ mod tests {let first_orderings = convert_to_orderings(first_child_orderings);let second_orderings = convert_to_orderings(second_child_orderings);let union_expected_orderings = convert_to_orderings(union_orderings);- let child1 = Arc::new(TestMemoryExec::update_cache(Arc::new(- TestMemoryExec::try_new(&[], Arc::clone(&schema), None)?- .try_with_sort_information(first_orderings)?,- )));- let child2 = Arc::new(TestMemoryExec::update_cache(Arc::new(- TestMemoryExec::try_new(&[], Arc::clone(&schema), None)?- .try_with_sort_information(second_orderings)?,- )));+ let child1_exec = TestMemoryExec::try_new(&[], Arc::clone(&schema), None)?+ .try_with_sort_information(first_orderings)?;+ let child1 = Arc::new(child1_exec);+ let child1 = Arc::new(TestMemoryExec::update_cache(&child1));+ let child2_exec = TestMemoryExec::try_new(&[], Arc::clone(&schema), None)?+ .try_with_sort_information(second_orderings)?;+ let child2 = Arc::new(child2_exec);+ let child2 = Arc::new(TestMemoryExec::update_cache(&child2));let mut union_expected_eq = EquivalenceProperties::new(Arc::clone(&schema));union_expected_eq.add_orderings(union_expected_orderings);diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rsindex 22132f2f86..3c999b1a40 100644--- a/datafusion/physical-plan/src/unnest.rs+++ b/datafusion/physical-plan/src/unnest.rs@@ -90,7 +90,7 @@ impl UnnestExec {&input,&list_column_indices,&struct_column_indices,- Arc::clone(&schema),+ &schema,)?;Ok(UnnestExec {@@ -109,7 +109,7 @@ impl UnnestExec {input: &Arc<dyn ExecutionPlan>,list_column_indices: &[ListUnnest],struct_column_indices: &[usize],- schema: SchemaRef,+ schema: &SchemaRef,) -> Result<PlanProperties> {// Find out which indices are not unnested, such that they can be copied over from the input planlet input_schema = input.schema();@@ -159,7 +159,7 @@ impl UnnestExec {// the unnest operation invalidates any global uniqueness or primary-key constraints.let input_eq_properties = input.equivalence_properties();let eq_properties = input_eq_properties- .project(&projection_mapping, Arc::clone(&schema))+ .project(&projection_mapping, Arc::clone(schema)).with_constraints(Constraints::default());// Output partitioning must use the projection mappingdiff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rsindex a59b2a5d1d..810c97cf47 100644--- a/datafusion/physical-plan/src/windows/window_agg_exec.rs+++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs@@ -82,7 +82,7 @@ impl WindowAggExec {let ordered_partition_by_indices =get_ordered_partition_by_indices(window_expr[0].partition_by(), &input)?;- let cache = Self::compute_properties(Arc::clone(&schema), &input, &window_expr)?;+ let cache = Self::compute_properties(&schema, &input, &window_expr)?;Ok(Self {input,window_expr,@@ -120,12 +120,12 @@ impl WindowAggExec {/// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc.fn compute_properties(- schema: SchemaRef,+ schema: &SchemaRef,input: &Arc<dyn ExecutionPlan>,window_exprs: &[Arc<dyn WindowExpr>],) -> Result<PlanProperties> {// Calculate equivalence properties:- let eq_properties = window_equivalence_properties(&schema, input, window_exprs)?;+ let eq_properties = window_equivalence_properties(schema, input, window_exprs)?;// Get output partitioning:// Because we can have repartitioning using the partition keys this