Apache DataFusion SQL Query Engine
chore: Update Rust toolchain to 1.98.1 (#25295)
Neil Conway6bbd3f42c3e8
diff --git a/Cargo.toml b/Cargo.tomlindex 05e1d22f58..5f11c5717c 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -283,7 +283,6 @@ enum_glob_use = "allow" # 98 hitsexplicit_into_iter_loop = "allow" # 55 hitsexplicit_iter_loop = "allow" # 189 hitsfloat_cmp = "allow" # 8 hits; exact float comparisons are often intentional here-from_iter_instead_of_collect = "allow" # 51 hitsif_not_else = "allow" # 133 hitsignored_unit_patterns = "allow" # 52 hitsimplicit_clone = "allow" # 198 hitsdiff --git a/datafusion-cli/src/helper.rs b/datafusion-cli/src/helper.rsindex e1e26701a2..c167e083d8 100644--- a/datafusion-cli/src/helper.rs+++ b/datafusion-cli/src/helper.rs@@ -161,7 +161,7 @@ impl Completer for CliHelper {if is_open_quote_for_location(line, pos) {self.completer.complete(line, pos, ctx)} else {- Ok((0, Vec::with_capacity(0)))+ Ok((0, Vec::new()))}}}diff --git a/datafusion-cli/src/object_storage.rs b/datafusion-cli/src/object_storage.rsindex 5e6337e303..2c9ca2c265 100644--- a/datafusion-cli/src/object_storage.rs+++ b/datafusion-cli/src/object_storage.rs@@ -58,6 +58,7 @@ use object_store::aws::resolve_bucket_region;#[cfg(test)]#[expect(clippy::unused_async,+ clippy::result_large_err,reason = "matches object_store::aws::resolve_bucket_region")]async fn resolve_bucket_region(diff --git a/datafusion-cli/src/object_storage/instrumented.rs b/datafusion-cli/src/object_storage/instrumented.rsindex a0321cacb3..5216ee4852 100644--- a/datafusion-cli/src/object_storage/instrumented.rs+++ b/datafusion-cli/src/object_storage/instrumented.rs@@ -143,6 +143,10 @@ pub struct InstrumentedObjectStore {requests: Arc<Mutex<Vec<RequestDetails>>>,}+#[expect(+ clippy::result_large_err,+ reason = "error type is dictated by the object_store API"+)]impl InstrumentedObjectStore {/// Returns a new [`InstrumentedObjectStore`] that wraps the provided [`ObjectStore`]fn new(object_store: Arc<dyn ObjectStore>, instrument_mode: AtomicU8) -> Self {diff --git a/datafusion-examples/examples/data_io/object_store_spill.rs b/datafusion-examples/examples/data_io/object_store_spill.rsindex c714c45570..f03d41367c 100644--- a/datafusion-examples/examples/data_io/object_store_spill.rs+++ b/datafusion-examples/examples/data_io/object_store_spill.rs@@ -242,6 +242,10 @@ struct ObjectStoreSpillWriter {}impl ObjectStoreSpillWriter {+ #[expect(+ clippy::result_large_err,+ reason = "error type is dictated by the object_store API"+ )]async fn flush_part(&mut self) -> object_store::Result<()> {if self.buffer.is_empty() {return Ok(());diff --git a/datafusion-examples/examples/data_io/remote_catalog.rs b/datafusion-examples/examples/data_io/remote_catalog.rsindex 49157c6b1f..22cdf39185 100644--- a/datafusion-examples/examples/data_io/remote_catalog.rs+++ b/datafusion-examples/examples/data_io/remote_catalog.rs@@ -130,7 +130,7 @@ struct RemoteCatalogInterface {}impl RemoteCatalogInterface {/// Establish a connection to the remote catalog- #[expect(clippy::unused_async)]+ #[expect(clippy::unused_async, clippy::unused_async_trait_impl)]pub async fn connect() -> Result<Self> {// In a real implementation this method might connect to a remote// catalog, validate credentials, cache basic information, etc@@ -138,7 +138,7 @@ impl RemoteCatalogInterface {}/// Fetches information for a specific table- #[expect(clippy::unused_async)]+ #[expect(clippy::unused_async, clippy::unused_async_trait_impl)]pub async fn table_info(&self, name: &str) -> Result<Option<SchemaRef>> {if name != "remote_table" {return Ok(None);@@ -157,7 +157,7 @@ impl RemoteCatalogInterface {}/// Fetches data for a table from a remote data source- #[expect(clippy::unused_async)]+ #[expect(clippy::unused_async, clippy::unused_async_trait_impl)]pub async fn read_data(&self, name: &str) -> Result<SendableRecordBatchStream> {if name != "remote_table" {return plan_err!("Remote table not found: {}", name);diff --git a/datafusion-examples/examples/flight/client.rs b/datafusion-examples/examples/flight/client.rsindex 8f6856a4e4..1cc7da2b93 100644--- a/datafusion-examples/examples/flight/client.rs+++ b/datafusion-examples/examples/flight/client.rs@@ -49,7 +49,7 @@ pub async fn client() -> Result<(), Box<dyn std::error::Error>> {let request = tonic::Request::new(FlightDescriptor {r#type: flight_descriptor::DescriptorType::Path as i32,cmd: Default::default(),- path: vec![format!("{}", parquet_temp.path_str()?)],+ path: vec![parquet_temp.path_str()?.to_string()],});let schema_result = client.get_schema(request).await?.into_inner();diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rsindex 2b8411ead6..a716ca7548 100644--- a/datafusion/common/src/hash_utils.rs+++ b/datafusion/common/src/hash_utils.rs@@ -1285,10 +1285,6 @@ mod tests {use std::hash::{BuildHasherDefault, Hasher};use std::sync::Arc;- use arrow::array::*;- #[cfg(not(feature = "force_hash_collisions"))]- use arrow::datatypes::*;-use super::*;#[cfg(not(feature = "force_hash_collisions"))]diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rsindex bad526a3a2..cf7047e598 100644--- a/datafusion/common/src/scalar/mod.rs+++ b/datafusion/common/src/scalar/mod.rs@@ -7921,7 +7921,7 @@ mod tests {#[allow(clippy::allow_attributes, clippy::mutable_key_type)]// ScalarValue has interior mutability but is intentionally used as hash key- let mut s = HashSet::with_capacity(0);+ let mut s = HashSet::new();// do NOT clone `sv` here because this may shrink the vector capacitys.insert(v.pop().unwrap());// hashsets may easily grow during insert, so capacity is dynamicdiff --git a/datafusion/common/src/utils/hex.rs b/datafusion/common/src/utils/hex.rsindex 8ff9a6d36d..716fc4cea2 100644--- a/datafusion/common/src/utils/hex.rs+++ b/datafusion/common/src/utils/hex.rs@@ -153,8 +153,8 @@ pub fn encode_bytes_to_slice(bytes: &[u8], case: HexCase, out: &mut [u8]) -> Res);}let lookup = case.lookup();- for (&b, chunk) in bytes.iter().zip(out.chunks_exact_mut(2)) {- chunk.copy_from_slice(&lookup[b as usize]);+ for (&b, chunk) in bytes.iter().zip(out.as_chunks_mut::<2>().0) {+ *chunk = lookup[b as usize];}Ok(())}diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rsindex 26e6e0c74c..7502b67774 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs@@ -1157,7 +1157,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {// Now check what our filter looks like#[cfg(not(feature = "force_hash_collisions"))]insta::assert_snapshot!(- format!("{}", format_plan_for_test(&plan)),+ format_plan_for_test(&plan),@r"- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- CoalescePartitionsExec@@ -1175,7 +1175,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {// joins or any scenario where all build-side data naturally lands in one partition.#[cfg(feature = "force_hash_collisions")]insta::assert_snapshot!(- format!("{}", format_plan_for_test(&plan)),+ format_plan_for_test(&plan),@r"- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- CoalescePartitionsExec@@ -1373,7 +1373,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_range_partitioned() {// Now check what our filter looks likeinsta::assert_snapshot!(- format!("{}", format_plan_for_test(&plan)),+ format_plan_for_test(&plan),@r"- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- CoalescePartitionsExec@@ -1502,7 +1502,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_collect_left() {// Now check what our filter looks likeinsta::assert_snapshot!(- format!("{}", format_plan_for_test(&plan)),+ format_plan_for_test(&plan),@r"- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- CoalescePartitionsExecdiff --git a/datafusion/datasource/src/boundary_stream.rs b/datafusion/datasource/src/boundary_stream.rsindex a1d10651b5..dbd41dfd6b 100644--- a/datafusion/datasource/src/boundary_stream.rs+++ b/datafusion/datasource/src/boundary_stream.rs@@ -84,6 +84,10 @@ pub struct AlignedBoundaryStream {}/// Fetch a bounded byte range from `store` and return it as a stream+#[expect(+ clippy::result_large_err,+ reason = "error type is dictated by the object_store API"+)]async fn get_stream(store: Arc<dyn ObjectStore>,location: object_store::path::Path,@@ -148,6 +152,10 @@ impl AlignedBoundaryStream {/// newline is not found within that window, `ScanningLastTerminator`/// automatically issues additional `END_SCAN_LOOKAHEAD`-sized GETs/// via `store` until the newline is found or EOF is reached.+ #[expect(+ clippy::result_large_err,+ reason = "error type is dictated by the object_store API"+ )]pub async fn new(store: Arc<dyn ObjectStore>,location: object_store::path::Path,diff --git a/datafusion/expr/src/logical_plan/display.rs b/datafusion/expr/src/logical_plan/display.rsindex 3009253f53..1d95bf894b 100644--- a/datafusion/expr/src/logical_plan/display.rs+++ b/datafusion/expr/src/logical_plan/display.rs@@ -426,7 +426,7 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {json!({"Node Type": "CopyTo","Output URL": output_url,- "File Type": format!("{}", file_type.get_ext()),+ "File Type": file_type.get_ext(),"Options": op_str})}@@ -490,7 +490,7 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {"Node Type": format!("{} Join", join_type),"Join Constraint": format!("{:?}", join_constraint),"Join Keys": join_expr.join(", "),- "Filter": format!("{}", filter_expr)+ "Filter": filter_expr.to_string()})}LogicalPlan::AsOfJoin(AsOfJoin {diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/accumulate.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/accumulate.rsindex 36ba0fba0b..bf5f37475f 100644--- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/accumulate.rs+++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/accumulate.rs@@ -443,14 +443,13 @@ pub fn accumulate<T, F>(let nulls = values.nulls().unwrap();// This is based on (ahem, COPY/PASTE) arrow::compute::aggregate::sum// iterate over in chunks of 64 bits for more efficient null checking- let group_indices_chunks = group_indices.chunks_exact(64);- let data_chunks = data.chunks_exact(64);+ let (group_indices_chunks, group_indices_remainder) =+ group_indices.as_chunks::<64>();+ let (data_chunks, data_remainder) = data.as_chunks::<64>();let bit_chunks = nulls.inner().bit_chunks();- let group_indices_remainder = group_indices_chunks.remainder();- let data_remainder = data_chunks.remainder();-group_indices_chunks+ .iter().zip(data_chunks).zip(bit_chunks.iter()).for_each(|((group_index_chunk, data_chunk), mask)| {@@ -605,13 +604,12 @@ pub fn accumulate_indices<F>(}(None, Some(filter)) => {debug_assert_eq!(filter.len(), group_indices.len());- let group_indices_chunks = group_indices.chunks_exact(64);+ let (group_indices_chunks, group_indices_remainder) =+ group_indices.as_chunks::<64>();let filter_validity = filter_to_validity(filter);let bit_chunks = filter_validity.bit_chunks();- let group_indices_remainder = group_indices_chunks.remainder();-- group_indices_chunks.zip(bit_chunks.iter()).for_each(+ group_indices_chunks.iter().zip(bit_chunks.iter()).for_each(|(group_index_chunk, mask)| {// index_mask has value 1 << i in the looplet mut index_mask = 1;@@ -642,12 +640,11 @@ pub fn accumulate_indices<F>(debug_assert_eq!(valids.len(), group_indices.len());// This is based on (ahem, COPY/PASTA) arrow::compute::aggregate::sum// iterate over in chunks of 64 bits for more efficient null checking- let group_indices_chunks = group_indices.chunks_exact(64);+ let (group_indices_chunks, group_indices_remainder) =+ group_indices.as_chunks::<64>();let bit_chunks = valids.inner().bit_chunks();- let group_indices_remainder = group_indices_chunks.remainder();-- group_indices_chunks.zip(bit_chunks.iter()).for_each(+ group_indices_chunks.iter().zip(bit_chunks.iter()).for_each(|(group_index_chunk, mask)| {// index_mask has value 1 << i in the looplet mut index_mask = 1;@@ -679,14 +676,14 @@ pub fn accumulate_indices<F>(debug_assert_eq!(filter.len(), group_indices.len());debug_assert_eq!(valids.len(), group_indices.len());- let group_indices_chunks = group_indices.chunks_exact(64);+ let (group_indices_chunks, group_indices_remainder) =+ group_indices.as_chunks::<64>();let valid_bit_chunks = valids.inner().bit_chunks();let filter_validity = filter_to_validity(filter);let filter_bit_chunks = filter_validity.bit_chunks();- let group_indices_remainder = group_indices_chunks.remainder();-group_indices_chunks+ .iter().zip(valid_bit_chunks.iter()).zip(filter_bit_chunks.iter()).for_each(|((group_index_chunk, valid_mask), filter_mask)| {diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rsindex 672150ee9b..d88fef3a9f 100644--- a/datafusion/functions-aggregate/src/approx_distinct.rs+++ b/datafusion/functions-aggregate/src/approx_distinct.rs@@ -359,9 +359,8 @@ impl GroupHll {);}let mut delta = 0;- for chunk in bytes.chunks_exact(size_of::<u64>()) {- let h = u64::from_le_bytes(chunk.try_into().unwrap());- delta += self.add_hash(h);+ for chunk in bytes.as_chunks::<{ size_of::<u64>() }>().0 {+ delta += self.add_hash(u64::from_le_bytes(*chunk));}Ok(delta)}diff --git a/datafusion/functions/src/core/getfield.rs b/datafusion/functions/src/core/getfield.rsindex a0f024bbc7..475c6518d2 100644--- a/datafusion/functions/src/core/getfield.rs+++ b/datafusion/functions/src/core/getfield.rs@@ -302,7 +302,7 @@ fn simplify_get_field_over_struct_constructor(args: &[Expr]) -> Option<Expr> {return None;}let mut matched = None;- for pair in ctor_args.chunks_exact(2) {+ for [name_expr, value_expr] in ctor_args.as_chunks::<2>().0 {// Every name must be a literal string: a non-literal name appearing// *before* the first match could evaluate to `field_name` at runtime// and become the real first match (Arrow's `column_by_name` returns@@ -313,13 +313,13 @@ fn simplify_get_field_over_struct_constructor(args: &[Expr]) -> Option<Expr> {// — it can never precede the first match — so bailing there is a// deliberate approximation we accept to keep this check simple, not a// correctness requirement.- let Expr::Literal(name, _) = &pair[0] else {+ let Expr::Literal(name, _) = name_expr else {return None;};let name = name.try_as_str().flatten()?;// `column_by_name` resolves to the first match, so do the same.if matched.is_none() && name == field_name {- matched = Some(&pair[1]);+ matched = Some(value_expr);}}matched?.clone()diff --git a/datafusion/functions/src/core/named_struct.rs b/datafusion/functions/src/core/named_struct.rsindex 71c48ce89e..e1aa441e0b 100644--- a/datafusion/functions/src/core/named_struct.rs+++ b/datafusion/functions/src/core/named_struct.rs@@ -162,8 +162,10 @@ impl ScalarUDFImpl for NamedStructFunc {let values: Vec<ColumnarValue> = args.args- .chunks_exact(2)- .map(|chunk| chunk[1].clone())+ .as_chunks::<2>()+ .0+ .iter()+ .map(|[_name, value]| value.clone()).collect();let arrays = ColumnarValue::values_to_arrays(&values)?;Ok(ColumnarValue::Array(Arc::new(StructArray::new(diff --git a/datafusion/physical-plan/src/repartition/distributor_channels.rs b/datafusion/physical-plan/src/repartition/distributor_channels.rsindex ce72eddf05..00894463b0 100644--- a/datafusion/physical-plan/src/repartition/distributor_channels.rs+++ b/datafusion/physical-plan/src/repartition/distributor_channels.rs@@ -232,7 +232,7 @@ impl<T> Future for SendFuture<'_, T> {this.gate.decr_empty_channels();guard_channel_state.take_recv_wakers()} else {- Vec::with_capacity(0)+ Vec::new()}};@@ -316,10 +316,10 @@ impl<T> Future for RecvFuture<'_, T> {if this.gate.empty_channels.load(Ordering::SeqCst) > 0 {guard.take().unwrap_or_default()} else {- Vec::with_capacity(0)+ Vec::new()}} else {- Vec::with_capacity(0)+ Vec::new()};drop(guard_channel_state);@@ -439,7 +439,7 @@ impl Gate {wake} else {- Vec::with_capacity(0)+ Vec::new()}};diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rsindex 92c9101afb..d5cbe64cae 100644--- a/datafusion/sql/src/unparser/expr.rs+++ b/datafusion/sql/src/unparser/expr.rs@@ -406,20 +406,18 @@ impl Unparser<'_> {..} = &agg.params;- let args_to_use;- let within_group;-// if this is a WITHIN GROUP aggregate, skip the prepended arg- if agg.func.supports_within_group_clause() && !order_by.is_empty() {- args_to_use = self.function_args_to_sql(&args[1..])?;- within_group = order_by- .iter()- .map(|sort_expr| self.sort_to_sql(sort_expr))- .collect::<Result<Vec<ast::OrderByExpr>>>()?;- } else {- args_to_use = self.function_args_to_sql(args)?;- within_group = Vec::new();- }+ let (args_to_use, within_group) =+ if agg.func.supports_within_group_clause() && !order_by.is_empty() {+ let args_to_use = self.function_args_to_sql(&args[1..])?;+ let within_group = order_by+ .iter()+ .map(|sort_expr| self.sort_to_sql(sort_expr))+ .collect::<Result<Vec<ast::OrderByExpr>>>()?;+ (args_to_use, within_group)+ } else {+ (self.function_args_to_sql(args)?, Vec::new())+ };let filter = match filter {Some(filter) => Some(Box::new(self.expr_to_sql_inner(filter)?)),@@ -742,16 +740,18 @@ impl Unparser<'_> {);let args = args- .chunks_exact(2)- .map(|chunk| {- let key = match &chunk[0] {+ .as_chunks::<2>()+ .0+ .iter()+ .map(|[name, value]| {+ let key = match name {Expr::Literal(ScalarValue::Utf8(Some(s)), _) => self.new_ident_quoted_if_needs(s.to_string()),- _ => return internal_err!("named_struct expects even arguments to be strings, but received: {:?}", &chunk[0])+ _ => return internal_err!("named_struct expects even arguments to be strings, but received: {name:?}")};Ok(ast::DictionaryField {key,- value: Box::new(self.expr_to_sql_with_nesting(&chunk[1])?),+ value: Box::new(self.expr_to_sql_with_nesting(value)?),})}).collect::<Result<Vec<_>>>()?;diff --git a/dev/depcheck/rust-toolchain.toml b/dev/depcheck/rust-toolchain.tomlindex 5639a821f5..8a06035a6a 100644--- a/dev/depcheck/rust-toolchain.toml+++ b/dev/depcheck/rust-toolchain.toml@@ -19,5 +19,5 @@# to compile this workspace and run CI jobs.[toolchain]-channel = "1.97.0"+channel = "1.98.1"components = ["rustfmt", "clippy"]diff --git a/docs/source/contributor-guide/development_environment.md b/docs/source/contributor-guide/development_environment.mdindex 5b6f024b53..20cfd6edd0 100644--- a/docs/source/contributor-guide/development_environment.md+++ b/docs/source/contributor-guide/development_environment.md@@ -108,7 +108,7 @@ DataFusion is written in Rust and it uses a standard rust toolkit:- `rustup update stable` DataFusion generally uses the latest stable release of Rust, though it may lag when new Rust toolchains release- See which toolchain is currently pinned in the [`rust-toolchain.toml`](https://github.com/apache/datafusion/blob/main/rust-toolchain.toml) file- - This can cause issues such as not having the rust-analyzer component installed for the specified toolchain, in which case just install it manually, e.g. `rustup component add --toolchain 1.97.0 rust-analyzer`+ - This can cause issues such as not having the rust-analyzer component installed for the specified toolchain, in which case just install it manually, e.g. `rustup component add --toolchain 1.98.1 rust-analyzer`- `cargo build`- `cargo fmt` to format the code- etc.diff --git a/rust-toolchain.toml b/rust-toolchain.tomlindex 5639a821f5..8a06035a6a 100644--- a/rust-toolchain.toml+++ b/rust-toolchain.toml@@ -19,5 +19,5 @@# to compile this workspace and run CI jobs.[toolchain]-channel = "1.97.0"+channel = "1.98.1"components = ["rustfmt", "clippy"]