Skip to content

Apache DataFusion SQL Query Engine

minor: refactor with `assert_or_internal_err!()` in `datafusion/physical-plan` (#18730)

Kumar Ujjawal85777c4f7ea0
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index 18090b6422..5fafce0bea 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -44,7 +44,10 @@ use arrow::datatypes::{Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use arrow_schema::FieldRef;
use datafusion_common::stats::Precision;
-use datafusion_common::{internal_err, not_impl_err, Constraint, Constraints, Result};
+use datafusion_common::{
+ assert_eq_or_internal_err, not_impl_err, Constraint, Constraints, DataFusionError,
+ Result,
+};
use datafusion_execution::TaskContext;
use datafusion_expr::{Accumulator, Aggregate};
use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
@@ -492,9 +495,13 @@ impl AggregateExec {
schema: SchemaRef,
) -> Result<Self> {
// Make sure arguments are consistent in size
- if aggr_expr.len() != filter_expr.len() {
- return internal_err!("Inconsistent aggregate expr: {:?} and filter expr: {:?} for AggregateExec, their size should match", aggr_expr, filter_expr);
- }
+ assert_eq_or_internal_err!(
+ aggr_expr.len(),
+ filter_expr.len(),
+ "Inconsistent aggregate expr: {:?} and filter expr: {:?} for AggregateExec, their size should match",
+ aggr_expr,
+ filter_expr
+ );
let input_eq_properties = input.equivalence_properties();
// Get GROUP BY expressions:
diff --git a/datafusion/physical-plan/src/aggregates/row_hash.rs b/datafusion/physical-plan/src/aggregates/row_hash.rs
index 3c6577af42..2e1b70da28 100644
--- a/datafusion/physical-plan/src/aggregates/row_hash.rs
+++ b/datafusion/physical-plan/src/aggregates/row_hash.rs
@@ -39,7 +39,10 @@ use crate::{RecordBatchStream, SendableRecordBatchStream};
use arrow::array::*;
use arrow::datatypes::SchemaRef;
-use datafusion_common::{internal_err, DataFusionError, Result};
+use datafusion_common::{
+ assert_eq_or_internal_err, assert_or_internal_err, internal_err, DataFusionError,
+ Result,
+};
use datafusion_execution::memory_pool::proxy::VecAllocExt;
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
use datafusion_execution::TaskContext;
@@ -938,9 +941,10 @@ impl GroupedHashAggregateStream {
)?;
}
_ => {
- if opt_filter.is_some() {
- return internal_err!("aggregate filter should be applied in partial stage, there should be no filter in final stage");
- }
+ assert_or_internal_err!(
+ opt_filter.is_none(),
+ "aggregate filter should be applied in partial stage, there should be no filter in final stage"
+ );
// if aggregation is over intermediate states,
// use merge
@@ -1218,9 +1222,11 @@ impl GroupedHashAggregateStream {
let input_values = evaluate_many(&self.aggregate_arguments, &batch)?;
let filter_values = evaluate_optional(&self.filter_expressions, &batch)?;
- if group_values.len() != 1 {
- return internal_err!("group_values expected to have single element");
- }
+ assert_eq_or_internal_err!(
+ group_values.len(),
+ 1,
+ "group_values expected to have single element"
+ );
let mut output = group_values.swap_remove(0);
let iter = self
diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rs
index c696cf5aa5..f0a3da7555 100644
--- a/datafusion/physical-plan/src/analyze.rs
+++ b/datafusion/physical-plan/src/analyze.rs
@@ -31,7 +31,7 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning};
use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};
use datafusion_common::instant::Instant;
-use datafusion_common::{internal_err, DataFusionError, Result};
+use datafusion_common::{assert_eq_or_internal_err, DataFusionError, Result};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
@@ -161,11 +161,11 @@ impl ExecutionPlan for AnalyzeExec {
partition: usize,
context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream> {
- if 0 != partition {
- return internal_err!(
- "AnalyzeExec invalid partition. Expected 0, got {partition}"
- );
- }
+ assert_eq_or_internal_err!(
+ partition,
+ 0,
+ "AnalyzeExec invalid partition. Expected 0, got {partition}"
+ );
// Gather futures that will run each input partition in
// parallel (on a separate tokio task) using a JoinSet to
diff --git a/datafusion/physical-plan/src/async_func.rs b/datafusion/physical-plan/src/async_func.rs
index 54a76e0ebb..fcb2ce54be 100644
--- a/datafusion/physical-plan/src/async_func.rs
+++ b/datafusion/physical-plan/src/async_func.rs
@@ -23,7 +23,7 @@ use crate::{
use arrow::array::RecordBatch;
use arrow_schema::{Fields, Schema, SchemaRef};
use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
-use datafusion_common::{internal_err, Result};
+use datafusion_common::{assert_eq_or_internal_err, DataFusionError, Result};
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_physical_expr::async_scalar_function::AsyncFuncExpr;
use datafusion_physical_expr::equivalence::ProjectionMapping;
@@ -148,9 +148,11 @@ impl ExecutionPlan for AsyncFuncExec {
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
) -> Result<Arc<dyn ExecutionPlan>> {
- if children.len() != 1 {
- return internal_err!("AsyncFuncExec wrong number of children");
- }
+ assert_eq_or_internal_err!(
+ children.len(),
+ 1,
+ "AsyncFuncExec wrong number of children"
+ );
Ok(Arc::new(AsyncFuncExec::try_new(
self.async_exprs.clone(),
Arc::clone(&children[0]),
diff --git a/datafusion/physical-plan/src/coalesce/mod.rs b/datafusion/physical-plan/src/coalesce/mod.rs
index 8405a660f0..860fc81bf6 100644
--- a/datafusion/physical-plan/src/coalesce/mod.rs
+++ b/datafusion/physical-plan/src/coalesce/mod.rs
@@ -18,7 +18,7 @@
use arrow::array::RecordBatch;
use arrow::compute::BatchCoalescer;
use arrow::datatypes::SchemaRef;
-use datafusion_common::{internal_err, Result};
+use datafusion_common::{assert_or_internal_err, DataFusionError, Result};
/// Concatenate multiple [`RecordBatch`]es and apply a limit
///
@@ -88,11 +88,10 @@ impl LimitedBatchCoalescer {
/// Returns an error if called after [`Self::finish`] or if the internal push
/// operation fails.
pub fn push_batch(&mut self, batch: RecordBatch) -> Result<PushBatchStatus> {
- if self.finished {
- return internal_err!(
- "LimitedBatchCoalescer: cannot push batch after finish"
- );
- }
+ assert_or_internal_err!(
+ !self.finished,
+ "LimitedBatchCoalescer: cannot push batch after finish"
+ );
// if we are at the limit, return LimitReached
if let Some(fetch) = self.fetch {
diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs
index 2597dc6408..dfb1800a2e 100644
--- a/datafusion/physical-plan/src/coalesce_partitions.rs
+++ b/datafusion/physical-plan/src/coalesce_partitions.rs
@@ -33,7 +33,9 @@ use crate::projection::{make_with_child, ProjectionExec};
use crate::{DisplayFormatType, ExecutionPlan, Partitioning};
use datafusion_common::config::ConfigOptions;
-use datafusion_common::{internal_err, Result};
+use datafusion_common::{
+ assert_eq_or_internal_err, internal_err, DataFusionError, Result,
+};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::PhysicalExpr;
@@ -160,9 +162,11 @@ impl ExecutionPlan for CoalescePartitionsExec {
context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream> {
// CoalescePartitionsExec produces a single partition
- if 0 != partition {
- return internal_err!("CoalescePartitionsExec invalid partition {partition}");
- }
+ assert_eq_or_internal_err!(
+ partition,
+ 0,
+ "CoalescePartitionsExec invalid partition {partition}"
+ );
let input_partitions = self.input.output_partitioning().partition_count();
match input_partitions {
diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rs
index b62d15e6d2..29d5ba1ca8 100644
--- a/datafusion/physical-plan/src/coop.rs
+++ b/datafusion/physical-plan/src/coop.rs
@@ -85,7 +85,7 @@ use crate::{
};
use arrow::record_batch::RecordBatch;
use arrow_schema::Schema;
-use datafusion_common::{internal_err, Result, Statistics};
+use datafusion_common::{assert_eq_or_internal_err, DataFusionError, Result, Statistics};
use datafusion_execution::TaskContext;
use crate::execution_plan::SchedulingType;
@@ -269,9 +269,11 @@ impl ExecutionPlan for CooperativeExec {
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
) -> Result<Arc<dyn ExecutionPlan>> {
- if children.len() != 1 {
- return internal_err!("CooperativeExec requires exactly one child");
- }
+ assert_eq_or_internal_err!(
+ children.len(),
+ 1,
+ "CooperativeExec requires exactly one child"
+ );
Ok(Arc::new(CooperativeExec::new(children.swap_remove(0))))
}
diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rs
index 40b4ec61dc..e9606fe1be 100644
--- a/datafusion/physical-plan/src/empty.rs
+++ b/datafusion/physical-plan/src/empty.rs
@@ -29,7 +29,7 @@ use crate::{
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
-use datafusion_common::{internal_err, Result};
+use datafusion_common::{assert_or_internal_err, DataFusionError, Result};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
@@ -136,13 +136,12 @@ impl ExecutionPlan for EmptyExec {
) -> Result<SendableRecordBatchStream> {
trace!("Start EmptyExec::execute for partition {} of context session_id {} and task_id {:?}", partition, context.session_id(), context.task_id());
- if partition >= self.partitions {
- return internal_err!(
- "EmptyExec invalid partition {} (expected less than {})",
- partition,
- self.partitions
- );
- }
+ assert_or_internal_err!(
+ partition < self.partitions,
+ "EmptyExec invalid partition {} (expected less than {})",
+ partition,
+ self.partitions
+ );
Ok(Box::pin(MemoryStream::try_new(
self.data()?,
@@ -157,13 +156,12 @@ impl ExecutionPlan for EmptyExec {
fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {
if let Some(partition) = partition {
- if partition >= self.partitions {
- return internal_err!(
- "EmptyExec invalid partition {} (expected less than {})",
- partition,
- self.partitions
- );
- }
+ assert_or_internal_err!(
+ partition < self.partitions,
+ "EmptyExec invalid partition {} (expected less than {})",
+ partition,
+ self.partitions
+ );
}
let batch = self
diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs
index ffa9611d26..245342f5cd 100644
--- a/datafusion/physical-plan/src/execution_plan.rs
+++ b/datafusion/physical-plan/src/execution_plan.rs
@@ -47,7 +47,10 @@ use crate::stream::RecordBatchStreamAdapter;
use arrow::array::{Array, RecordBatch};
use arrow::datatypes::SchemaRef;
use datafusion_common::config::ConfigOptions;
-use datafusion_common::{exec_err, Constraints, DataFusionError, Result};
+use datafusion_common::{
+ assert_eq_or_internal_err, assert_or_internal_err, exec_err, Constraints,
+ DataFusionError, Result,
+};
use datafusion_common_runtime::JoinSet;
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
@@ -484,13 +487,12 @@ pub trait ExecutionPlan: Debug + DisplayAs + Send + Sync {
if let Some(idx) = partition {
// Validate partition index
let partition_count = self.properties().partitioning.partition_count();
- if idx >= partition_count {
- return internal_err!(
- "Invalid partition index: {}, the partition count is {}",
- idx,
- partition_count
- );
- }
+ assert_or_internal_err!(
+ idx < partition_count,
+ "Invalid partition index: {}, the partition count is {}",
+ idx,
+ partition_count
+ );
}
Ok(Statistics::new_unknown(&self.schema()))
}
@@ -1082,15 +1084,15 @@ impl PlanProperties {
macro_rules! check_len {
($target:expr, $func_name:ident, $expected_len:expr) => {
let actual_len = $target.$func_name().len();
- if actual_len != $expected_len {
- return internal_err!(
- "{}::{} returned Vec with incorrect size: {} != {}",
- $target.name(),
- stringify!($func_name),
- actual_len,
- $expected_len
- );
- }
+ assert_eq_or_internal_err!(
+ actual_len,
+ $expected_len,
+ "{}::{} returned Vec with incorrect size: {} != {}",
+ $target.name(),
+ stringify!($func_name),
+ actual_len,
+ $expected_len
+ );
};
}
@@ -1127,9 +1129,12 @@ pub fn with_new_children_if_necessary(
children: Vec<Arc<dyn ExecutionPlan>>,
) -> Result<Arc<dyn ExecutionPlan>> {
let old_children = plan.children();
- if children.len() != old_children.len() {
- internal_err!("Wrong number of children")
- } else if children.is_empty()
+ assert_eq_or_internal_err!(
+ children.len(),
+ old_children.len(),
+ "Wrong number of children"
+ );
+ if children.is_empty()
|| children
.iter()
.zip(old_children.iter())
diff --git a/datafusion/physical-plan/src/explain.rs b/datafusion/physical-plan/src/explain.rs
index bf488ccfae..91e2593ec4 100644
--- a/datafusion/physical-plan/src/explain.rs
+++ b/datafusion/physical-plan/src/explain.rs
@@ -27,7 +27,7 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning};
use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};
use datafusion_common::display::StringifiedPlan;
-use datafusion_common::{internal_err, Result};
+use datafusion_common::{assert_eq_or_internal_err, DataFusionError, Result};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
@@ -134,9 +134,11 @@ impl ExecutionPlan for ExplainExec {
context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream> {
trace!("Start ExplainExec::execute for partition {} of context session_id {} and task_id {:?}", partition, context.session_id(), context.task_id());
- if 0 != partition {
- return internal_err!("ExplainExec invalid partition {partition}");
- }
+ assert_eq_or_internal_err!(
+ partition,
+ 0,
+ "ExplainExec invalid partition {partition}"
+ );
let mut type_builder =
StringBuilder::with_capacity(self.stringified_plans.len(), 1024);
let mut plan_builder =
diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs
index 2c531786c9..42a76c6d51 100644
--- a/datafusion/physical-plan/src/joins/cross_join.rs
+++ b/datafusion/physical-plan/src/joins/cross_join.rs
@@ -41,7 +41,10 @@ use arrow::array::{RecordBatch, RecordBatchOptions};
use arrow::compute::concat_batches;
use arrow::datatypes::{Fields, Schema, SchemaRef};
use datafusion_common::stats::Precision;
-use datafusion_common::{internal_err, JoinType, Result, ScalarValue};
+use datafusion_common::{
+ assert_eq_or_internal_err, internal_err, DataFusionError, JoinType, Result,
+ ScalarValue,
+};
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::equivalence::join_equivalence_properties;
@@ -300,12 +303,12 @@ impl ExecutionPlan for CrossJoinExec {
partition: usize,
context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream> {
- if self.left.output_partitioning().partition_count() != 1 {
- return internal_err!(
- "Invalid CrossJoinExec, the output partition count of the left child must be 1,\
+ assert_eq_or_internal_err!(
+ self.left.output_partitioning().partition_count(),
+ 1,
+ "Invalid CrossJoinExec, the output partition count of the left child must be 1,\
consider using CoalescePartitionsExec or the EnforceDistribution rule"
- );
- }
+ );
let stream = self.right.execute(partition, Arc::clone(&context))?;
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index 43df83b6b4..d923acaea0 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -63,7 +63,8 @@ use arrow_schema::DataType;
use datafusion_common::config::ConfigOptions;
use datafusion_common::utils::memory::estimate_memory_size;
use datafusion_common::{
- internal_err, plan_err, project_schema, JoinSide, JoinType, NullEquality, Result,
+ assert_or_internal_err, plan_err, project_schema, DataFusionError, JoinSide,
+ JoinType, NullEquality, Result,
};
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
use datafusion_execution::TaskContext;
@@ -912,20 +913,18 @@ impl ExecutionPlan for HashJoinExec {
let left_partitions = self.left.output_partitioning().partition_count();
let right_partitions = self.right.output_partitioning().partition_count();
- if self.mode == PartitionMode::Partitioned && left_partitions != right_partitions
- {
- return internal_err!(
- "Invalid HashJoinExec, partition count mismatch {left_partitions}!={right_partitions},\
- consider using RepartitionExec"
- );
- }
+ assert_or_internal_err!(
+ self.mode != PartitionMode::Partitioned
+ || left_partitions == right_partitions,
+ "Invalid HashJoinExec, partition count mismatch {left_partitions}!={right_partitions},\
+ consider using RepartitionExec"
+ );
- if self.mode == PartitionMode::CollectLeft && left_partitions != 1 {
- return internal_err!(
- "Invalid HashJoinExec, the output partition count of the left child must be 1 in CollectLeft mode,\
- consider using CoalescePartitionsExec or the EnforceDistribution rule"
- );
- }
+ assert_or_internal_err!(
+ self.mode != PartitionMode::CollectLeft || left_partitions == 1,
+ "Invalid HashJoinExec, the output partition count of the left child must be 1 in CollectLeft mode,\
+ consider using CoalescePartitionsExec or the EnforceDistribution rule"
+ );
let enable_dynamic_filter_pushdown = self.dynamic_filter.is_some();
@@ -1512,7 +1511,7 @@ mod tests {
use datafusion_common::test_util::{batches_to_sort_string, batches_to_string};
use datafusion_common::{
assert_batches_eq, assert_batches_sorted_eq, assert_contains, exec_err,
- ScalarValue,
+ internal_err, ScalarValue,
};
use datafusion_execution::config::SessionConfig;
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs
index a6b3df8819..26533ce4c7 100644
--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs
+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs
@@ -61,8 +61,9 @@ use arrow::record_batch::RecordBatch;
use arrow_schema::DataType;
use datafusion_common::cast::as_boolean_array;
use datafusion_common::{
- arrow_err, internal_datafusion_err, internal_err, project_schema,
- unwrap_or_internal_err, DataFusionError, JoinSide, Result, ScalarValue, Statistics,
+ arrow_err, assert_eq_or_internal_err, internal_datafusion_err, internal_err,
+ project_schema, unwrap_or_internal_err, DataFusionError, JoinSide, Result,
+ ScalarValue, Statistics,
};
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
use datafusion_execution::TaskContext;
@@ -491,12 +492,12 @@ impl ExecutionPlan for NestedLoopJoinExec {
partition: usize,
context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream> {
- if self.left.output_partitioning().partition_count() != 1 {
- return internal_err!(
- "Invalid NestedLoopJoinExec, the output partition count of the left child must be 1,\
+ assert_eq_or_internal_err!(
+ self.left.output_partitioning().partition_count(),
+ 1,
+ "Invalid NestedLoopJoinExec, the output partition count of the left child must be 1,\
consider using CoalescePartitionsExec or the EnforceDistribution rule"
- );
- }
+ );
let metrics = NestedLoopJoinMetrics::new(&self.metrics, partition);
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
index 592878a3bb..bc139f43b7 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
@@ -45,7 +45,8 @@ use crate::{
use arrow::compute::SortOptions;
use arrow::datatypes::SchemaRef;
use datafusion_common::{
- internal_err, plan_err, JoinSide, JoinType, NullEquality, Result,
+ assert_eq_or_internal_err, internal_err, plan_err, DataFusionError, JoinSide,
+ JoinType, NullEquality, Result,
};
use datafusion_execution::memory_pool::MemoryConsumer;
use datafusion_execution::TaskContext;
@@ -459,12 +460,12 @@ impl ExecutionPlan for SortMergeJoinExec {
) -> Result<SendableRecordBatchStream> {
let left_partitions = self.left.output_partitioning().partition_count();
let right_partitions = self.right.output_partitioning().partition_count();
- if left_partitions != right_partitions {
- return internal_err!(
- "Invalid SortMergeJoinExec, partition count mismatch {left_partitions}!={right_partitions},\
+ assert_eq_or_internal_err!(
+ left_partitions,
+ right_partitions,
+ "Invalid SortMergeJoinExec, partition count mismatch {left_partitions}!={right_partitions},\
consider using RepartitionExec"
- );
- }
+ );
let (on_left, on_right) = self.on.iter().cloned().unzip();
let (streamed, buffered, on_streamed, on_buffered) =
if SortMergeJoinExec::probe_side(&self.join_type) == JoinSide::Left {
diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
index 4b18bdca1b..5a2c80e8fd 100644
--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
@@ -67,7 +67,8 @@ use arrow::record_batch::RecordBatch;
use datafusion_common::hash_utils::create_hashes;
use datafusion_common::utils::bisect;
use datafusion_common::{
- internal_err, plan_err, HashSet, JoinSide, JoinType, NullEquality, Result,
+ assert_eq_or_internal_err, plan_err, DataFusionError, HashSet, JoinSide, JoinType,
+ NullEquality, Result,
};
use datafusion_execution::memory_pool::MemoryConsumer;
use datafusion_execution::TaskContext;
@@ -481,12 +482,12 @@ impl ExecutionPlan for SymmetricHashJoinExec {
) -> Result<SendableRecordBatchStream> {
let left_partitions = self.left.output_partitioning().partition_count();
let right_partitions = self.right.output_partitioning().partition_count();
- if left_partitions != right_partitions {
- return internal_err!(
- "Invalid SymmetricHashJoinExec, partition count mismatch {left_partitions}!={right_partitions},\
+ assert_eq_or_internal_err!(
+ left_partitions,
+ right_partitions,
+ "Invalid SymmetricHashJoinExec, partition count mismatch {left_partitions}!={right_partitions},\
consider using RepartitionExec"
- );
- }
+ );
// If `filter_state` and `filter` are both present, then calculate sorted
// filter expressions for both sides, and build an expression graph.
let (left_sorted_filter_expr, right_sorted_filter_expr, graph) = match (
diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs
index 6a0cae20e5..a78de53fcc 100644
--- a/datafusion/physical-plan/src/limit.rs
+++ b/datafusion/physical-plan/src/limit.rs
@@ -32,7 +32,9 @@ use crate::{DisplayFormatType, Distribution, ExecutionPlan, Partitioning};
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
-use datafusion_common::{internal_err, Result};
+use datafusion_common::{
+ assert_eq_or_internal_err, internal_err, DataFusionError, Result,
+};
use datafusion_execution::TaskContext;
use futures::stream::{Stream, StreamExt};
@@ -167,14 +169,18 @@ impl ExecutionPlan for GlobalLimitExec {
) -> Result<SendableRecordBatchStream> {
trace!("Start GlobalLimitExec::execute for partition: {partition}");
// GlobalLimitExec has a single output partition
- if 0 != partition {
- return internal_err!("GlobalLimitExec invalid partition {partition}");
- }
+ assert_eq_or_internal_err!(
+ partition,
+ 0,
+ "GlobalLimitExec invalid partition {partition}"
+ );
// GlobalLimitExec requires a single input partition
- if 1 != self.input.output_partitioning().partition_count() {
- return internal_err!("GlobalLimitExec requires a single input partition");
- }
+ assert_eq_or_internal_err!(
+ self.input.output_partitioning().partition_count(),
+ 1,
+ "GlobalLimitExec requires a single input partition"
+ );
let baseline_metrics = BaselineMetrics::new(&self.metrics, partition);
let stream = self.input.execute(0, context)?;
diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rs
index 09710ae1e2..97a748e33a 100644
--- a/datafusion/physical-plan/src/memory.rs
+++ b/datafusion/physical-plan/src/memory.rs
@@ -32,7 +32,9 @@ use crate::{
use arrow::array::RecordBatch;
use arrow::datatypes::SchemaRef;
-use datafusion_common::{internal_err, Result};
+use datafusion_common::{
+ assert_eq_or_internal_err, assert_or_internal_err, DataFusionError, Result,
+};
use datafusion_execution::memory_pool::MemoryReservation;
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
@@ -224,16 +226,17 @@ impl LazyMemoryExec {
}
pub fn try_set_partitioning(&mut self, partitioning: Partitioning) -> Result<()> {
- if partitioning.partition_count() != self.batch_generators.len() {
- internal_err!(
- "Partition count must match generator count: {} != {}",
- partitioning.partition_count(),
- self.batch_generators.len()
- )
- } else {
- self.cache.partitioning = partitioning;
- Ok(())
- }
+ let partition_count = partitioning.partition_count();
+ let generator_count = self.batch_generators.len();
+ assert_eq_or_internal_err!(
+ partition_count,
+ generator_count,
+ "Partition count must match generator count: {} != {}",
+ partition_count,
+ generator_count
+ );
+ self.cache.partitioning = partitioning;
+ Ok(())
}
pub fn add_ordering(&mut self, ordering: impl IntoIterator<Item = PhysicalSortExpr>) {
@@ -314,11 +317,11 @@ impl ExecutionPlan for LazyMemoryExec {
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
) -> Result<Arc<dyn ExecutionPlan>> {
- if children.is_empty() {
- Ok(self)
- } else {
- internal_err!("Children cannot be replaced in LazyMemoryExec")
- }
+ assert_or_internal_err!(
+ children.is_empty(),
+ "Children cannot be replaced in LazyMemoryExec"
+ );
+ Ok(self)
}
fn execute(
@@ -326,13 +329,12 @@ impl ExecutionPlan for LazyMemoryExec {
partition: usize,
_context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream> {
- if partition >= self.batch_generators.len() {
- return internal_err!(
- "Invalid partition {} for LazyMemoryExec with {} partitions",
- partition,
- self.batch_generators.len()
- );
- }
+ assert_or_internal_err!(
+ partition < self.batch_generators.len(),
+ "Invalid partition {} for LazyMemoryExec with {} partitions",
+ partition,
+ self.batch_generators.len()
+ );
let baseline_metrics = BaselineMetrics::new(&self.metrics, partition);
diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rs
index e7df79f867..d657128ee1 100644
--- a/datafusion/physical-plan/src/placeholder_row.rs
+++ b/datafusion/physical-plan/src/placeholder_row.rs
@@ -30,7 +30,7 @@ use crate::{
use arrow::array::{ArrayRef, NullArray, RecordBatch, RecordBatchOptions};
use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef};
-use datafusion_common::{internal_err, Result};
+use datafusion_common::{assert_or_internal_err, DataFusionError, Result};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
@@ -154,13 +154,11 @@ impl ExecutionPlan for PlaceholderRowExec {
) -> Result<SendableRecordBatchStream> {
trace!("Start PlaceholderRowExec::execute for partition {} of context session_id {} and task_id {:?}", partition, context.session_id(), context.task_id());
- if partition >= self.partitions {
- return internal_err!(
- "PlaceholderRowExec invalid partition {} (expected less than {})",
- partition,
- self.partitions
- );
- }
+ assert_or_internal_err!(
+ partition < self.partitions,
+ "PlaceholderRowExec invalid partition {partition} (expected less than {})",
+ self.partitions
+ );
let ms = MemoryStream::try_new(self.data()?, Arc::clone(&self.schema), None)?;
Ok(Box::pin(cooperative(ms)))
diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs
index 6d69f47efa..53cd24185f 100644
--- a/datafusion/physical-plan/src/repartition/mod.rs
+++ b/datafusion/physical-plan/src/repartition/mod.rs
@@ -46,8 +46,10 @@ use arrow::datatypes::{SchemaRef, UInt32Type};
use datafusion_common::config::ConfigOptions;
use datafusion_common::stats::Precision;
use datafusion_common::utils::transpose;
-use datafusion_common::{internal_err, ColumnStatistics, HashMap};
-use datafusion_common::{not_impl_err, DataFusionError, Result};
+use datafusion_common::{
+ assert_or_internal_err, internal_err, ColumnStatistics, DataFusionError, HashMap,
+};
+use datafusion_common::{not_impl_err, Result};
use datafusion_common_runtime::SpawnedTask;
use datafusion_execution::memory_pool::MemoryConsumer;
use datafusion_execution::TaskContext;
@@ -976,13 +978,12 @@ impl ExecutionPlan for RepartitionExec {
return Ok(Statistics::new_unknown(&self.schema()));
}
- if partition >= partition_count {
- return internal_err!(
- "RepartitionExec invalid partition {} (expected less than {})",
- partition,
- self.partitioning().partition_count()
- );
- }
+ assert_or_internal_err!(
+ partition < partition_count,
+ "RepartitionExec invalid partition {} (expected less than {})",
+ partition,
+ partition_count
+ );
let mut stats = self.input.partition_statistics(None)?;
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs
index 2b31ff3da9..c4c871da74 100644
--- a/datafusion/physical-plan/src/sorts/sort.rs
+++ b/datafusion/physical-plan/src/sorts/sort.rs
@@ -57,8 +57,8 @@ use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays};
use arrow::datatypes::SchemaRef;
use datafusion_common::config::SpillCompression;
use datafusion_common::{
- internal_datafusion_err, internal_err, unwrap_or_internal_err, DataFusionError,
- Result,
+ assert_or_internal_err, internal_datafusion_err, unwrap_or_internal_err,
+ DataFusionError, Result,
};
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
use datafusion_execution::runtime_env::RuntimeEnv;
@@ -425,9 +425,10 @@ impl ExternalSorter {
(*max_record_batch_size).max(batch.get_sliced_size()?);
}
- if !globally_sorted_batches.is_empty() {
- return internal_err!("This function consumes globally_sorted_batches, so it should be empty after taking.");
- }
+ assert_or_internal_err!(
+ globally_sorted_batches.is_empty(),
+ "This function consumes globally_sorted_batches, so it should be empty after taking."
+ );
Ok(())
}
@@ -518,11 +519,10 @@ impl ExternalSorter {
/// Sorts the in-memory batches and merges them into a single sorted run, then writes
/// the result to spill files.
async fn sort_and_spill_in_mem_batches(&mut self) -> Result<()> {
- if self.in_mem_batches.is_empty() {
- return internal_err!(
- "in_mem_batches must not be empty when attempting to sort and spill"
- );
- }
+ assert_or_internal_err!(
+ !self.in_mem_batches.is_empty(),
+ "in_mem_batches must not be empty when attempting to sort and spill"
+ );
// Release the memory reserved for merge back to the pool so
// there is some left when `in_mem_sort_stream` requests an
@@ -534,11 +534,10 @@ impl ExternalSorter {
self.in_mem_sort_stream(self.metrics.baseline.intermediate())?;
// After `in_mem_sort_stream()` is constructed, all `in_mem_batches` is taken
// to construct a globally sorted stream.
- if !self.in_mem_batches.is_empty() {
- return internal_err!(
- "in_mem_batches should be empty after constructing sorted stream"
- );
- }
+ assert_or_internal_err!(
+ self.in_mem_batches.is_empty(),
+ "in_mem_batches should be empty after constructing sorted stream"
+ );
// 'global' here refers to all buffered batches when the memory limit is
// reached. This variable will buffer the sorted batches after
// sort-preserving merge and incrementally append to spill files.
@@ -570,11 +569,10 @@ impl ExternalSorter {
// Sanity check after spilling
let buffers_cleared_property =
self.in_mem_batches.is_empty() && globally_sorted_batches.is_empty();
- if !buffers_cleared_property {
- return internal_err!(
- "in_mem_batches and globally_sorted_batches should be cleared before"
- );
- }
+ assert_or_internal_err!(
+ buffers_cleared_property,
+ "in_mem_batches and globally_sorted_batches should be cleared before"
+ );
// Reserve headroom for next sort/merge
self.reserve_memory_for_merge()?;
diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
index 3a94f156fa..34b6d5108f 100644
--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
@@ -30,7 +30,9 @@ use crate::{
Partitioning, PlanProperties, SendableRecordBatchStream, Statistics,
};
-use datafusion_common::{internal_err, Result};
+use datafusion_common::{
+ assert_eq_or_internal_err, internal_err, DataFusionError, Result,
+};
use datafusion_execution::memory_pool::MemoryConsumer;
use datafusion_execution::TaskContext;
use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequirements};
@@ -281,11 +283,11 @@ impl ExecutionPlan for SortPreservingMergeExec {
context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream> {
trace!("Start SortPreservingMergeExec::execute for partition: {partition}");
- if 0 != partition {
- return internal_err!(
- "SortPreservingMergeExec invalid partition {partition}"
- );
- }
+ assert_eq_or_internal_err!(
+ partition,
+ 0,
+ "SortPreservingMergeExec invalid partition {partition}"
+ );
let input_partitions = self.input.output_partitioning().partition_count();
trace!(
diff --git a/datafusion/physical-plan/src/sorts/streaming_merge.rs b/datafusion/physical-plan/src/sorts/streaming_merge.rs
index 191b135753..ea0358a55f 100644
--- a/datafusion/physical-plan/src/sorts/streaming_merge.rs
+++ b/datafusion/physical-plan/src/sorts/streaming_merge.rs
@@ -27,7 +27,7 @@ use crate::sorts::{
use crate::{SendableRecordBatchStream, SpillManager};
use arrow::array::*;
use arrow::datatypes::{DataType, SchemaRef};
-use datafusion_common::{internal_err, Result};
+use datafusion_common::{assert_or_internal_err, internal_err, DataFusionError, Result};
use datafusion_execution::disk_manager::RefCountedTempFile;
use datafusion_execution::memory_pool::{
human_readable_size, MemoryConsumer, MemoryPool, MemoryReservation,
@@ -213,11 +213,10 @@ impl<'a> StreamingMergeBuilder<'a> {
}
// Early return if streams are empty:
- if streams.is_empty() {
- return internal_err!(
- "Streams/sorted spill files cannot be empty for streaming merge"
- );
- }
+ assert_or_internal_err!(
+ !streams.is_empty(),
+ "Streams/sorted spill files cannot be empty for streaming merge"
+ );
// Unwrapping mandatory fields
let schema = schema.expect("Schema cannot be empty for streaming merge");
diff --git a/datafusion/physical-plan/src/test.rs b/datafusion/physical-plan/src/test.rs
index 349f9955b6..bc32192917 100644
--- a/datafusion/physical-plan/src/test.rs
+++ b/datafusion/physical-plan/src/test.rs
@@ -37,7 +37,8 @@ use crate::{DisplayAs, DisplayFormatType, PlanProperties};
use arrow::array::{Array, ArrayRef, Int32Array, RecordBatch};
use arrow_schema::{DataType, Field, Schema, SchemaRef};
use datafusion_common::{
- config::ConfigOptions, internal_err, project_schema, Result, Statistics,
+ assert_or_internal_err, config::ConfigOptions, project_schema, DataFusionError,
+ Result, Statistics,
};
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_physical_expr::equivalence::{
@@ -317,12 +318,11 @@ impl TestMemoryExec {
.map(|field| field.name() != col.name())
.unwrap_or(true)
});
- if let Some(col) = ambiguous_column {
- return internal_err!(
- "Column {:?} is not found in the original schema of the TestMemoryExec",
- col
- );
- }
+ assert_or_internal_err!(
+ ambiguous_column.is_none(),
+ "Column {:?} is not found in the original schema of the TestMemoryExec",
+ ambiguous_column.as_ref().unwrap()
+ );
// If there is a projection on the source, we also need to project orderings
if let Some(projection) = &self.projection {
diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs
index c95678dac9..1432c2b09b 100644
--- a/datafusion/physical-plan/src/union.rs
+++ b/datafusion/physical-plan/src/union.rs
@@ -45,7 +45,9 @@ use arrow::datatypes::{Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use datafusion_common::config::ConfigOptions;
use datafusion_common::stats::Precision;
-use datafusion_common::{exec_err, internal_datafusion_err, internal_err, Result};
+use datafusion_common::{
+ assert_or_internal_err, exec_err, internal_datafusion_err, DataFusionError, Result,
+};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::{calculate_union, EquivalenceProperties, PhysicalExpr};
@@ -410,11 +412,10 @@ pub struct InterleaveExec {
impl InterleaveExec {
/// Create a new InterleaveExec
pub fn try_new(inputs: Vec<Arc<dyn ExecutionPlan>>) -> Result<Self> {
- if !can_interleave(inputs.iter()) {
- return internal_err!(
- "Not all InterleaveExec children have a consistent hash partitioning"
- );
- }
+ assert_or_internal_err!(
+ can_interleave(inputs.iter()),
+ "Not all InterleaveExec children have a consistent hash partitioning"
+ );
let cache = Self::compute_properties(&inputs)?;
Ok(InterleaveExec {
inputs,
@@ -485,11 +486,10 @@ impl ExecutionPlan for InterleaveExec {
children: Vec<Arc<dyn ExecutionPlan>>,
) -> Result<Arc<dyn ExecutionPlan>> {
// New children are no longer interleavable, which might be a bug of optimization rewrite.
- if !can_interleave(children.iter()) {
- return internal_err!(
- "Can not create InterleaveExec: new children can not be interleaved"
- );
- }
+ assert_or_internal_err!(
+ can_interleave(children.iter()),
+ "Can not create InterleaveExec: new children can not be interleaved"
+ );
Ok(Arc::new(InterleaveExec::try_new(children)?))
}
diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rs
index 1b7cb9bb76..a59b2a5d1d 100644
--- a/datafusion/physical-plan/src/windows/window_agg_exec.rs
+++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs
@@ -42,7 +42,7 @@ use arrow::error::ArrowError;
use arrow::record_batch::RecordBatch;
use datafusion_common::stats::Precision;
use datafusion_common::utils::{evaluate_partition_ranges, transpose};
-use datafusion_common::{internal_err, Result};
+use datafusion_common::{assert_eq_or_internal_err, DataFusionError, Result};
use datafusion_execution::TaskContext;
use datafusion_physical_expr_common::sort_expr::{
OrderingRequirements, PhysicalSortExpr,
@@ -337,9 +337,11 @@ impl WindowAggStream {
ordered_partition_by_indices: Vec<usize>,
) -> Result<Self> {
// In WindowAggExec all partition by columns should be ordered.
- if window_expr[0].partition_by().len() != ordered_partition_by_indices.len() {
- return internal_err!("All partition by columns should have an ordering");
- }
+ assert_eq_or_internal_err!(
+ window_expr[0].partition_by().len(),
+ ordered_partition_by_indices.len(),
+ "All partition by columns should have an ordering"
+ );
Ok(Self {
schema,
input,
diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rs
index 40a22f94b8..a77e7b2cf1 100644
--- a/datafusion/physical-plan/src/work_table.rs
+++ b/datafusion/physical-plan/src/work_table.rs
@@ -31,7 +31,9 @@ use crate::{
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
-use datafusion_common::{internal_datafusion_err, internal_err, Result};
+use datafusion_common::{
+ assert_eq_or_internal_err, internal_datafusion_err, DataFusionError, Result,
+};
use datafusion_execution::memory_pool::MemoryReservation;
use datafusion_execution::TaskContext;
use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
@@ -192,11 +194,11 @@ impl ExecutionPlan for WorkTableExec {
_context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream> {
// WorkTable streams must be the plan base.
- if partition != 0 {
- return internal_err!(
- "WorkTableExec got an invalid partition {partition} (expected 0)"
- );
- }
+ assert_eq_or_internal_err!(
+ partition,
+ 0,
+ "WorkTableExec got an invalid partition {partition} (expected 0)"
+ );
let batch = self.work_table.take()?;
let stream =