Apache DataFusion SQL Query Engine
Feat/update rust 2024 edition (#18915)
Tim Saucer36ec9f1de0ae
diff --git a/datafusion/common-runtime/Cargo.toml b/datafusion/common-runtime/Cargo.tomlindex fd9a818bcb..65e1544601 100644--- a/datafusion/common-runtime/Cargo.toml+++ b/datafusion/common-runtime/Cargo.toml@@ -21,7 +21,7 @@ description = "Common Runtime functionality for DataFusion query engine"keywords = ["arrow", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = { workspace = true }+edition = "2024"homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/common-runtime/src/common.rs b/datafusion/common-runtime/src/common.rsindex cebd6e04cd..649abb338c 100644--- a/datafusion/common-runtime/src/common.rs+++ b/datafusion/common-runtime/src/common.rs@@ -115,7 +115,7 @@ impl<R> Drop for SpawnedTask<R> {mod tests {use super::*;- use std::future::{pending, Pending};+ use std::future::{Pending, pending};use tokio::{runtime::Runtime, sync::oneshot};diff --git a/datafusion/common-runtime/src/lib.rs b/datafusion/common-runtime/src/lib.rsindex eb32fead7e..81fd61a4c6 100644--- a/datafusion/common-runtime/src/lib.rs+++ b/datafusion/common-runtime/src/lib.rs@@ -34,5 +34,5 @@ mod trace_utils;pub use common::SpawnedTask;pub use join_set::JoinSet;pub use trace_utils::{- set_join_set_tracer, trace_block, trace_future, JoinSetTracer, JoinSetTracerError,+ JoinSetTracer, JoinSetTracerError, set_join_set_tracer, trace_block, trace_future,};diff --git a/datafusion/common-runtime/src/trace_utils.rs b/datafusion/common-runtime/src/trace_utils.rsindex c3a39c355f..f8adbe8825 100644--- a/datafusion/common-runtime/src/trace_utils.rs+++ b/datafusion/common-runtime/src/trace_utils.rs@@ -15,8 +15,8 @@// specific language governing permissions and limitations// under the License.-use futures::future::BoxFuture;use futures::FutureExt;+use futures::future::BoxFuture;use std::any::Any;use std::error::Error;use std::fmt::{Display, Formatter, Result as FmtResult};diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.tomlindex 5c36e33c72..dac06de659 100644--- a/datafusion/common/Cargo.toml+++ b/datafusion/common/Cargo.toml@@ -21,7 +21,7 @@ description = "Common functionality for DataFusion query engine"keywords = ["arrow", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = { workspace = true }+edition = "2024"homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/common/src/cast.rs b/datafusion/common/src/cast.rsindex b95167ca13..29082cc303 100644--- a/datafusion/common/src/cast.rs+++ b/datafusion/common/src/cast.rs@@ -20,11 +20,11 @@//! but provide an error message rather than a panic, as the corresponding//! kernels in arrow-rs such as `as_boolean_array` do.-use crate::{downcast_value, Result};+use crate::{Result, downcast_value};use arrow::array::{BinaryViewArray, Decimal32Array, Decimal64Array, DurationMicrosecondArray,DurationMillisecondArray, DurationNanosecondArray, DurationSecondArray, Float16Array,- Int16Array, Int8Array, LargeBinaryArray, LargeListViewArray, LargeStringArray,+ Int8Array, Int16Array, LargeBinaryArray, LargeListViewArray, LargeStringArray,ListViewArray, StringViewArray, UInt16Array,};use arrow::{@@ -37,8 +37,8 @@ use arrow::{MapArray, NullArray, OffsetSizeTrait, PrimitiveArray, StringArray, StructArray,Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray,Time64NanosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray,- TimestampNanosecondArray, TimestampSecondArray, UInt32Array, UInt64Array,- UInt8Array, UnionArray,+ TimestampNanosecondArray, TimestampSecondArray, UInt8Array, UInt32Array,+ UInt64Array, UnionArray,},datatypes::{ArrowDictionaryKeyType, ArrowPrimitiveType},};diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex 9ba2550c39..a4385f0fd8 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -1121,7 +1121,7 @@ impl<'a> TryInto<arrow::util::display::FormatOptions<'a>> for &'a FormatOptionsreturn _config_err!("Invalid duration format: {}. Valid values are pretty or iso8601",self.duration_format- )+ );}};@@ -2257,13 +2257,13 @@ impl ConfigField for TableParquetOptions {[_meta] | [_meta, ""] => {return _config_err!("Invalid metadata key provided, missing key in metadata::<key>"- )+ );}[_meta, k] => k.into(),_ => {return _config_err!("Invalid metadata key provided, found too many '::' in \"{key}\""- )+ );}};self.key_value_metadata.insert(k, Some(value.into()));@@ -3287,9 +3287,11 @@ mod tests {.set("format.bloom_filter_enabled::col1", "true").unwrap();let entries = table_config.entries();- assert!(entries- .iter()- .any(|item| item.key == "format.bloom_filter_enabled::col1"))+ assert!(+ entries+ .iter()+ .any(|item| item.key == "format.bloom_filter_enabled::col1")+ )}#[cfg(feature = "parquet")]@@ -3303,10 +3305,10 @@ mod tests {).unwrap();let entries = table_parquet_options.entries();- assert!(entries- .iter()- .any(|item| item.key- == "crypto.file_encryption.column_key_as_hex::double_field"))+ assert!(+ entries.iter().any(|item| item.key+ == "crypto.file_encryption.column_key_as_hex::double_field")+ )}#[cfg(feature = "parquet")]diff --git a/datafusion/common/src/cse.rs b/datafusion/common/src/cse.rsindex 674d338617..93169d6a02 100644--- a/datafusion/common/src/cse.rs+++ b/datafusion/common/src/cse.rs@@ -19,12 +19,12 @@//! a [`CSEController`], that defines how to eliminate common subtrees from a particular//! [`TreeNode`] tree.+use crate::Result;use crate::hash_utils::combine_hashes;use crate::tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter,TreeNodeVisitor,};-use crate::Result;use indexmap::IndexMap;use std::collections::HashMap;use std::hash::{BuildHasher, Hash, Hasher, RandomState};@@ -676,13 +676,13 @@ where#[cfg(test)]mod test {+ use crate::Result;use crate::alias::AliasGenerator;use crate::cse::{- CSEController, HashNode, IdArray, Identifier, NodeStats, NormalizeEq,- Normalizeable, CSE,+ CSE, CSEController, HashNode, IdArray, Identifier, NodeStats, NormalizeEq,+ Normalizeable,};use crate::tree_node::tests::TestTreeNode;- use crate::Result;use std::collections::HashSet;use std::hash::{Hash, Hasher};diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rsindex 24d152a7db..fd4584bca0 100644--- a/datafusion/common/src/dfschema.rs+++ b/datafusion/common/src/dfschema.rs@@ -23,10 +23,10 @@ use std::fmt::{Display, Formatter};use std::hash::Hash;use std::sync::Arc;-use crate::error::{DataFusionError, Result, _plan_err, _schema_err};+use crate::error::{_plan_err, _schema_err, DataFusionError, Result};use crate::{- field_not_found, unqualified_field_not_found, Column, FunctionalDependencies,- SchemaError, TableReference,+ Column, FunctionalDependencies, SchemaError, TableReference, field_not_found,+ unqualified_field_not_found,};use arrow::compute::can_cast_types;@@ -982,36 +982,35 @@ fn format_field_with_indent(result.push_str(&format!("{indent}|-- {field_name}: map (nullable = {nullable_str})\n"));- if let DataType::Struct(inner_fields) = field.data_type() {- if inner_fields.len() == 2 {- format_field_with_indent(- result,- "key",- inner_fields[0].data_type(),- inner_fields[0].is_nullable(),- &child_indent,- );- let value_contains_null =- field.is_nullable().to_string().to_lowercase();- // Handle complex value types properly- match inner_fields[1].data_type() {- DataType::Struct(_)- | DataType::List(_)- | DataType::LargeList(_)- | DataType::FixedSizeList(_, _)- | DataType::Map(_, _) => {- format_field_with_indent(- result,- "value",- inner_fields[1].data_type(),- inner_fields[1].is_nullable(),- &child_indent,- );- }- _ => {- result.push_str(&format!("{child_indent}|-- value: {} (nullable = {value_contains_null})\n",+ if let DataType::Struct(inner_fields) = field.data_type()+ && inner_fields.len() == 2+ {+ format_field_with_indent(+ result,+ "key",+ inner_fields[0].data_type(),+ inner_fields[0].is_nullable(),+ &child_indent,+ );+ let value_contains_null = field.is_nullable().to_string().to_lowercase();+ // Handle complex value types properly+ match inner_fields[1].data_type() {+ DataType::Struct(_)+ | DataType::List(_)+ | DataType::LargeList(_)+ | DataType::FixedSizeList(_, _)+ | DataType::Map(_, _) => {+ format_field_with_indent(+ result,+ "value",+ inner_fields[1].data_type(),+ inner_fields[1].is_nullable(),+ &child_indent,+ );+ }+ _ => {+ result.push_str(&format!("{child_indent}|-- value: {} (nullable = {value_contains_null})\n",format_simple_data_type(inner_fields[1].data_type())));- }}}}@@ -1433,12 +1432,14 @@ mod tests {join.to_string());// test valid access- assert!(join- .field_with_qualified_name(&TableReference::bare("t1"), "c0")- .is_ok());- assert!(join- .field_with_qualified_name(&TableReference::bare("t2"), "c0")- .is_ok());+ assert!(+ join.field_with_qualified_name(&TableReference::bare("t1"), "c0")+ .is_ok()+ );+ assert!(+ join.field_with_qualified_name(&TableReference::bare("t2"), "c0")+ .is_ok()+ );// test invalid accessassert!(join.field_with_unqualified_name("c0").is_err());assert!(join.field_with_unqualified_name("t1.c0").is_err());@@ -1480,18 +1481,20 @@ mod tests {join.to_string());// test valid access- assert!(join- .field_with_qualified_name(&TableReference::bare("t1"), "c0")- .is_ok());+ assert!(+ join.field_with_qualified_name(&TableReference::bare("t1"), "c0")+ .is_ok()+ );assert!(join.field_with_unqualified_name("c0").is_ok());assert!(join.field_with_unqualified_name("c100").is_ok());assert!(join.field_with_name(None, "c100").is_ok());// test invalid accessassert!(join.field_with_unqualified_name("t1.c0").is_err());assert!(join.field_with_unqualified_name("t1.c100").is_err());- assert!(join- .field_with_qualified_name(&TableReference::bare(""), "c100")- .is_err());+ assert!(+ join.field_with_qualified_name(&TableReference::bare(""), "c100")+ .is_err()+ );Ok(())}@@ -1500,9 +1503,11 @@ mod tests {let left = DFSchema::try_from_qualified_schema("t1", &test_schema_1())?;let right = DFSchema::try_from(test_schema_1())?;let join = left.join(&right);- assert_contains!(join.unwrap_err().to_string(),- "Schema error: Schema contains qualified \- field name t1.c0 and unqualified field name c0 which would be ambiguous");+ assert_contains!(+ join.unwrap_err().to_string(),+ "Schema error: Schema contains qualified \+ field name t1.c0 and unqualified field name c0 which would be ambiguous"+ );Ok(())}diff --git a/datafusion/common/src/error.rs b/datafusion/common/src/error.rsindex 6a5cb31fe3..83ffca99d1 100644--- a/datafusion/common/src/error.rs+++ b/datafusion/common/src/error.rs@@ -1223,9 +1223,10 @@ mod test {#[test]fn datafusion_error_to_arrow() {let res = return_arrow_error().unwrap_err();- assert!(res- .to_string()- .starts_with("External error: Error during planning: foo"));+ assert!(+ res.to_string()+ .starts_with("External error: Error during planning: foo")+ );}#[test]@@ -1254,12 +1255,13 @@ mod test {.unwrap(),&"Error during planning: Err");- assert!(!err- .split(DataFusionError::BACK_TRACE_SEP)- .collect::<Vec<&str>>()- .get(1)- .unwrap()- .is_empty());+ assert!(+ !err.split(DataFusionError::BACK_TRACE_SEP)+ .collect::<Vec<&str>>()+ .get(1)+ .unwrap()+ .is_empty()+ );}#[cfg(not(feature = "backtrace"))]@@ -1403,9 +1405,11 @@ mod test {let external_error_2: DataFusionError = generic_error_2.into();println!("{external_error_2}");- assert!(external_error_2- .to_string()- .starts_with("External error: io error"));+ assert!(+ external_error_2+ .to_string()+ .starts_with("External error: io error")+ );}/// Model what happens when implementing SendableRecordBatchStream:diff --git a/datafusion/common/src/file_options/mod.rs b/datafusion/common/src/file_options/mod.rsindex 02667e0165..c7374949ec 100644--- a/datafusion/common/src/file_options/mod.rs+++ b/datafusion/common/src/file_options/mod.rs@@ -31,10 +31,10 @@ mod tests {use std::collections::HashMap;use crate::{+ Result,config::{ConfigFileType, TableOptions},file_options::{csv_writer::CsvWriterOptions, json_writer::JsonWriterOptions},parsers::CompressionTypeVariant,- Result,};use parquet::{diff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rsindex 564929c61b..47feb12c12 100644--- a/datafusion/common/src/file_options/parquet_writer.rs+++ b/datafusion/common/src/file_options/parquet_writer.rs@@ -20,8 +20,8 @@use std::sync::Arc;use crate::{+ _internal_datafusion_err, DataFusionError, Result,config::{ParquetOptions, TableParquetOptions},- DataFusionError, Result, _internal_datafusion_err,};use arrow::datatypes::Schema;@@ -34,8 +34,8 @@ use parquet::{file::{metadata::KeyValue,properties::{- EnabledStatistics, WriterProperties, WriterPropertiesBuilder, WriterVersion,- DEFAULT_STATISTICS_ENABLED,+ DEFAULT_STATISTICS_ENABLED, EnabledStatistics, WriterProperties,+ WriterPropertiesBuilder, WriterVersion,},},schema::types::ColumnPath,@@ -106,7 +106,9 @@ impl TryFrom<&TableParquetOptions> for WriterPropertiesBuilder {if !global.skip_arrow_metadata&& !key_value_metadata.contains_key(ARROW_SCHEMA_META_KEY){- return Err(_internal_datafusion_err!("arrow schema was not added to the kv_metadata, even though it is required by configuration settings"));+ return Err(_internal_datafusion_err!(+ "arrow schema was not added to the kv_metadata, even though it is required by configuration settings"+ ));}// add kv_meta, if any@@ -408,8 +410,8 @@ mod tests {};use parquet::basic::Compression;use parquet::file::properties::{- BloomFilterProperties, EnabledStatistics, DEFAULT_BLOOM_FILTER_FPP,- DEFAULT_BLOOM_FILTER_NDV,+ BloomFilterProperties, DEFAULT_BLOOM_FILTER_FPP, DEFAULT_BLOOM_FILTER_NDV,+ EnabledStatistics,};use std::collections::HashMap;@@ -674,8 +676,7 @@ mod tests {let mut default_table_writer_opts = TableParquetOptions::default();let default_parquet_opts = ParquetOptions::default();assert_eq!(- default_table_writer_opts.global,- default_parquet_opts,+ default_table_writer_opts.global, default_parquet_opts,"should have matching defaults for TableParquetOptions.global and ParquetOptions",);@@ -699,7 +700,9 @@ mod tests {"should have different created_by sources",);assert!(- default_writer_props.created_by().starts_with("parquet-rs version"),+ default_writer_props+ .created_by()+ .starts_with("parquet-rs version"),"should indicate that writer_props defaults came from the extern parquet crate",);assert!(@@ -733,8 +736,7 @@ mod tests {from_extern_parquet.global.skip_arrow_metadata = true;assert_eq!(- default_table_writer_opts,- from_extern_parquet,+ default_table_writer_opts, from_extern_parquet,"the default writer_props should have the same configuration as the session's default TableParquetOptions",);}diff --git a/datafusion/common/src/format.rs b/datafusion/common/src/format.rsindex 764190e118..a505bd0e1c 100644--- a/datafusion/common/src/format.rs+++ b/datafusion/common/src/format.rs@@ -176,9 +176,9 @@ impl FromStr for ExplainFormat {"tree" => Ok(ExplainFormat::Tree),"pgjson" => Ok(ExplainFormat::PostgresJSON),"graphviz" => Ok(ExplainFormat::Graphviz),- _ => {- Err(DataFusionError::Configuration(format!("Invalid explain format. Expected 'indent', 'tree', 'pgjson' or 'graphviz'. Got '{format}'")))- }+ _ => Err(DataFusionError::Configuration(format!(+ "Invalid explain format. Expected 'indent', 'tree', 'pgjson' or 'graphviz'. Got '{format}'"+ ))),}}}diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rsindex c6bd53356f..80299d9c0a 100644--- a/datafusion/common/src/hash_utils.rs+++ b/datafusion/common/src/hash_utils.rs@@ -1181,10 +1181,12 @@ mod tests {let result = with_hashes(empty, &random_state, |_hashes| Ok(()));assert!(result.is_err());- assert!(result- .unwrap_err()- .to_string()- .contains("requires at least one array"));+ assert!(+ result+ .unwrap_err()+ .to_string()+ .contains("requires at least one array")+ );}#[test]diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rsindex 15538e68d9..9e0238f97d 100644--- a/datafusion/common/src/lib.rs+++ b/datafusion/common/src/lib.rs@@ -67,21 +67,21 @@ pub mod utils;pub use arrow;pub use column::Column;pub use dfschema::{- qualified_name, DFSchema, DFSchemaRef, ExprSchema, SchemaExt, ToDFSchema,+ DFSchema, DFSchemaRef, ExprSchema, SchemaExt, ToDFSchema, qualified_name,};pub use diagnostic::Diagnostic;pub use error::{- field_not_found, unqualified_field_not_found, DataFusionError, Result, SchemaError,- SharedResult,+ DataFusionError, Result, SchemaError, SharedResult, field_not_found,+ unqualified_field_not_found,};pub use file_options::file_type::{- GetExt, DEFAULT_ARROW_EXTENSION, DEFAULT_AVRO_EXTENSION, DEFAULT_CSV_EXTENSION,- DEFAULT_JSON_EXTENSION, DEFAULT_PARQUET_EXTENSION,+ DEFAULT_ARROW_EXTENSION, DEFAULT_AVRO_EXTENSION, DEFAULT_CSV_EXTENSION,+ DEFAULT_JSON_EXTENSION, DEFAULT_PARQUET_EXTENSION, GetExt,};pub use functional_dependencies::{+ Constraint, Constraints, Dependency, FunctionalDependence, FunctionalDependencies,aggregate_functional_dependencies, get_required_group_by_exprs_indices,- get_target_functional_dependencies, Constraint, Constraints, Dependency,- FunctionalDependence, FunctionalDependencies,+ get_target_functional_dependencies,};use hashbrown::hash_map::DefaultHashBuilder;pub use join_type::{JoinConstraint, JoinSide, JoinType};@@ -137,10 +137,10 @@ macro_rules! downcast_value {// Not public API.#[doc(hidden)]pub mod __private {- use crate::error::_internal_datafusion_err;use crate::Result;+ use crate::error::_internal_datafusion_err;use arrow::array::Array;- use std::any::{type_name, Any};+ use std::any::{Any, type_name};#[doc(hidden)]pub trait DowncastArrayHelper {@@ -191,7 +191,7 @@ mod tests {assert_starts_with(error.to_string(),- "Internal error: could not cast array of type Int32 to arrow_array::array::primitive_array::PrimitiveArray<arrow_array::types::UInt64Type>"+ "Internal error: could not cast array of type Int32 to arrow_array::array::primitive_array::PrimitiveArray<arrow_array::types::UInt64Type>",);}diff --git a/datafusion/common/src/metadata.rs b/datafusion/common/src/metadata.rsindex 3a10cc2b42..df5e70ff5b 100644--- a/datafusion/common/src/metadata.rs+++ b/datafusion/common/src/metadata.rs@@ -20,7 +20,7 @@ use std::{collections::BTreeMap, sync::Arc};use arrow::datatypes::{DataType, Field};use hashbrown::HashMap;-use crate::{error::_plan_err, DataFusionError, ScalarValue};+use crate::{DataFusionError, ScalarValue, error::_plan_err};/// A [`ScalarValue`] with optional [`FieldMetadata`]#[derive(Debug, Clone)]diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rsindex d43816f75b..086d96e852 100644--- a/datafusion/common/src/nested_struct.rs+++ b/datafusion/common/src/nested_struct.rs@@ -15,10 +15,10 @@// specific language governing permissions and limitations// under the License.-use crate::error::{Result, _plan_err};+use crate::error::{_plan_err, Result};use arrow::{- array::{new_null_array, Array, ArrayRef, StructArray},- compute::{cast_with_options, CastOptions},+ array::{Array, ArrayRef, StructArray, new_null_array},+ compute::{CastOptions, cast_with_options},datatypes::{DataType::Struct, Field, FieldRef},};use std::sync::Arc;diff --git a/datafusion/common/src/param_value.rs b/datafusion/common/src/param_value.rsindex ebf68e4dd2..0fac6b529e 100644--- a/datafusion/common/src/param_value.rs+++ b/datafusion/common/src/param_value.rs@@ -16,7 +16,7 @@// under the License.use crate::error::{_plan_datafusion_err, _plan_err};-use crate::metadata::{check_metadata_with_storage_equal, ScalarAndMetadata};+use crate::metadata::{ScalarAndMetadata, check_metadata_with_storage_equal};use crate::{Result, ScalarValue};use arrow::datatypes::{DataType, Field, FieldRef};use std::collections::HashMap;diff --git a/datafusion/common/src/pruning.rs b/datafusion/common/src/pruning.rsindex 48750e3c99..4307b4d786 100644--- a/datafusion/common/src/pruning.rs+++ b/datafusion/common/src/pruning.rs@@ -485,8 +485,8 @@ impl PruningStatistics for CompositePruningStatistics {#[cfg(test)]mod tests {use crate::{- cast::{as_int32_array, as_uint64_array},ColumnStatistics,+ cast::{as_int32_array, as_uint64_array},};use super::*;diff --git a/datafusion/common/src/rounding.rs b/datafusion/common/src/rounding.rsindex 95eefd3235..1796143d7c 100644--- a/datafusion/common/src/rounding.rs+++ b/datafusion/common/src/rounding.rs@@ -47,7 +47,7 @@ extern crate libc;any(target_arch = "x86_64", target_arch = "aarch64"),not(target_os = "windows")))]-extern "C" {+unsafe extern "C" {fn fesetround(round: i32);fn fegetround() -> i32;}diff --git a/datafusion/common/src/scalar/cache.rs b/datafusion/common/src/scalar/cache.rsindex f1476a5187..5b1ad4e4ed 100644--- a/datafusion/common/src/scalar/cache.rs+++ b/datafusion/common/src/scalar/cache.rs@@ -20,10 +20,10 @@use std::iter::repeat_n;use std::sync::{Arc, LazyLock, Mutex};-use arrow::array::{new_null_array, Array, ArrayRef, PrimitiveArray};+use arrow::array::{Array, ArrayRef, PrimitiveArray, new_null_array};use arrow::datatypes::{- ArrowDictionaryKeyType, DataType, Int16Type, Int32Type, Int64Type, Int8Type,- UInt16Type, UInt32Type, UInt64Type, UInt8Type,+ ArrowDictionaryKeyType, DataType, Int8Type, Int16Type, Int32Type, Int64Type,+ UInt8Type, UInt16Type, UInt32Type, UInt64Type,};/// Maximum number of rows to cache to be conservative on memory usagediff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rsindex 3fd5a37224..94f2ba0ecf 100644--- a/datafusion/common/src/scalar/mod.rs+++ b/datafusion/common/src/scalar/mod.rs@@ -36,57 +36,57 @@ use std::sync::Arc;use crate::assert_or_internal_err;use crate::cast::{as_binary_array, as_binary_view_array, as_boolean_array, as_date32_array,- as_date64_array, as_decimal128_array, as_decimal256_array, as_decimal32_array,- as_decimal64_array, as_dictionary_array, as_duration_microsecond_array,+ as_date64_array, as_decimal32_array, as_decimal64_array, as_decimal128_array,+ as_decimal256_array, as_dictionary_array, as_duration_microsecond_array,as_duration_millisecond_array, as_duration_nanosecond_array,as_duration_second_array, as_fixed_size_binary_array, as_fixed_size_list_array,- as_float16_array, as_float32_array, as_float64_array, as_int16_array, as_int32_array,- as_int64_array, as_int8_array, as_interval_dt_array, as_interval_mdn_array,+ as_float16_array, as_float32_array, as_float64_array, as_int8_array, as_int16_array,+ as_int32_array, as_int64_array, as_interval_dt_array, as_interval_mdn_array,as_interval_ym_array, as_large_binary_array, as_large_list_array,as_large_string_array, as_string_array, as_string_view_array,as_time32_millisecond_array, as_time32_second_array, as_time64_microsecond_array,as_time64_nanosecond_array, as_timestamp_microsecond_array,as_timestamp_millisecond_array, as_timestamp_nanosecond_array,- as_timestamp_second_array, as_uint16_array, as_uint32_array, as_uint64_array,- as_uint8_array, as_union_array,+ as_timestamp_second_array, as_uint8_array, as_uint16_array, as_uint32_array,+ as_uint64_array, as_union_array,};-use crate::error::{DataFusionError, Result, _exec_err, _internal_err, _not_impl_err};+use crate::error::{_exec_err, _internal_err, _not_impl_err, DataFusionError, Result};use crate::format::DEFAULT_CAST_OPTIONS;use crate::hash_utils::create_hashes;use crate::utils::SingleRowListArrayBuilder;use crate::{_internal_datafusion_err, arrow_datafusion_err};use arrow::array::{- new_empty_array, new_null_array, Array, ArrayData, ArrayRef, ArrowNativeTypeOp,- ArrowPrimitiveType, AsArray, BinaryArray, BinaryViewArray, BooleanArray, Date32Array,- Date64Array, Decimal128Array, Decimal256Array, Decimal32Array, Decimal64Array,- DictionaryArray, DurationMicrosecondArray, DurationMillisecondArray,- DurationNanosecondArray, DurationSecondArray, FixedSizeBinaryArray,- FixedSizeListArray, Float16Array, Float32Array, Float64Array, GenericListArray,- Int16Array, Int32Array, Int64Array, Int8Array, IntervalDayTimeArray,- IntervalMonthDayNanoArray, IntervalYearMonthArray, LargeBinaryArray, LargeListArray,- LargeStringArray, ListArray, MapArray, MutableArrayData, OffsetSizeTrait,- PrimitiveArray, Scalar, StringArray, StringViewArray, StructArray,- Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray,- Time64NanosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray,- TimestampNanosecondArray, TimestampSecondArray, UInt16Array, UInt32Array,- UInt64Array, UInt8Array, UnionArray,+ Array, ArrayData, ArrayRef, ArrowNativeTypeOp, ArrowPrimitiveType, AsArray,+ BinaryArray, BinaryViewArray, BooleanArray, Date32Array, Date64Array, Decimal32Array,+ Decimal64Array, Decimal128Array, Decimal256Array, DictionaryArray,+ DurationMicrosecondArray, DurationMillisecondArray, DurationNanosecondArray,+ DurationSecondArray, FixedSizeBinaryArray, FixedSizeListArray, Float16Array,+ Float32Array, Float64Array, GenericListArray, Int8Array, Int16Array, Int32Array,+ Int64Array, IntervalDayTimeArray, IntervalMonthDayNanoArray, IntervalYearMonthArray,+ LargeBinaryArray, LargeListArray, LargeStringArray, ListArray, MapArray,+ MutableArrayData, OffsetSizeTrait, PrimitiveArray, Scalar, StringArray,+ StringViewArray, StructArray, Time32MillisecondArray, Time32SecondArray,+ Time64MicrosecondArray, Time64NanosecondArray, TimestampMicrosecondArray,+ TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray,+ UInt8Array, UInt16Array, UInt32Array, UInt64Array, UnionArray, new_empty_array,+ new_null_array,};use arrow::buffer::{BooleanBuffer, ScalarBuffer};-use arrow::compute::kernels::cast::{cast_with_options, CastOptions};+use arrow::compute::kernels::cast::{CastOptions, cast_with_options};use arrow::compute::kernels::numeric::{add, add_wrapping, div, mul, mul_wrapping, rem, sub, sub_wrapping,};use arrow::datatypes::{- i256, validate_decimal_precision_and_scale, ArrowDictionaryKeyType, ArrowNativeType,- ArrowTimestampType, DataType, Date32Type, Decimal128Type, Decimal256Type,- Decimal32Type, Decimal64Type, DecimalType, Field, Float32Type, Int16Type, Int32Type,- Int64Type, Int8Type, IntervalDayTime, IntervalDayTimeType, IntervalMonthDayNano,+ ArrowDictionaryKeyType, ArrowNativeType, ArrowTimestampType,+ DECIMAL128_MAX_PRECISION, DataType, Date32Type, Decimal32Type, Decimal64Type,+ Decimal128Type, Decimal256Type, DecimalType, Field, Float32Type, Int8Type, Int16Type,+ Int32Type, Int64Type, IntervalDayTime, IntervalDayTimeType, IntervalMonthDayNano,IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType, TimeUnit,TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,- TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type, UnionFields,- UnionMode, DECIMAL128_MAX_PRECISION,+ TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, UnionFields,+ UnionMode, i256, validate_decimal_precision_and_scale,};-use arrow::util::display::{array_value_to_string, ArrayFormatter, FormatOptions};+use arrow::util::display::{ArrayFormatter, FormatOptions, array_value_to_string};use cache::{get_or_create_cached_key_array, get_or_create_cached_null_array};use chrono::{Duration, NaiveDate};use half::f16;@@ -720,11 +720,7 @@ impl PartialOrd for ScalarValue {(Union(_, _, _), _) => None,(Dictionary(k1, v1), Dictionary(k2, v2)) => {// Don't compare if the key types don't match (it is effectively a different datatype)- if k1 == k2 {- v1.partial_cmp(v2)- } else {- None- }+ if k1 == k2 { v1.partial_cmp(v2) } else { None }}(Dictionary(_, _), _) => None,(Null, Null) => Some(Ordering::Equal),@@ -744,7 +740,9 @@ fn first_array_for_list(arr: &dyn Array) -> ArrayRef {} else if let Some(arr) = arr.as_fixed_size_list_opt() {arr.value(0)} else {- unreachable!("Since only List / LargeList / FixedSizeList are supported, this should never happen")+ unreachable!(+ "Since only List / LargeList / FixedSizeList are supported, this should never happen"+ )}}@@ -2359,18 +2357,20 @@ impl ScalarValue {macro_rules! build_array_primitive {($ARRAY_TY:ident, $SCALAR_TY:ident) => {{{- let array = scalars.map(|sv| {- if let ScalarValue::$SCALAR_TY(v) = sv {- Ok(v)- } else {- _exec_err!(- "Inconsistent types in ScalarValue::iter_to_array. \+ let array = scalars+ .map(|sv| {+ if let ScalarValue::$SCALAR_TY(v) = sv {+ Ok(v)+ } else {+ _exec_err!(+ "Inconsistent types in ScalarValue::iter_to_array. \Expected {:?}, got {:?}",- data_type, sv- )- }- })- .collect::<Result<$ARRAY_TY>>()?;+ data_type,+ sv+ )+ }+ })+ .collect::<Result<$ARRAY_TY>>()?;Arc::new(array)}}};@@ -2379,18 +2379,20 @@ impl ScalarValue {macro_rules! build_array_primitive_tz {($ARRAY_TY:ident, $SCALAR_TY:ident, $TZ:expr) => {{{- let array = scalars.map(|sv| {- if let ScalarValue::$SCALAR_TY(v, _) = sv {- Ok(v)- } else {- _exec_err!(- "Inconsistent types in ScalarValue::iter_to_array. \+ let array = scalars+ .map(|sv| {+ if let ScalarValue::$SCALAR_TY(v, _) = sv {+ Ok(v)+ } else {+ _exec_err!(+ "Inconsistent types in ScalarValue::iter_to_array. \Expected {:?}, got {:?}",- data_type, sv- )- }- })- .collect::<Result<$ARRAY_TY>>()?;+ data_type,+ sv+ )+ }+ })+ .collect::<Result<$ARRAY_TY>>()?;Arc::new(array.with_timezone_opt($TZ.clone()))}}};@@ -2401,18 +2403,20 @@ impl ScalarValue {macro_rules! build_array_string {($ARRAY_TY:ident, $SCALAR_TY:ident) => {{{- let array = scalars.map(|sv| {- if let ScalarValue::$SCALAR_TY(v) = sv {- Ok(v)- } else {- _exec_err!(- "Inconsistent types in ScalarValue::iter_to_array. \+ let array = scalars+ .map(|sv| {+ if let ScalarValue::$SCALAR_TY(v) = sv {+ Ok(v)+ } else {+ _exec_err!(+ "Inconsistent types in ScalarValue::iter_to_array. \Expected {:?}, got {:?}",- data_type, sv- )- }- })- .collect::<Result<$ARRAY_TY>>()?;+ data_type,+ sv+ )+ }+ })+ .collect::<Result<$ARRAY_TY>>()?;Arc::new(array)}}};@@ -3718,10 +3722,9 @@ impl ScalarValue {) -> Result<Self> {let source_type = self.data_type();if let Some(multiplier) = date_to_timestamp_multiplier(&source_type, target_type)+ && let Some(value) = self.date_scalar_value_as_i64(){- if let Some(value) = self.date_scalar_value_as_i64() {- ensure_timestamp_in_bounds(value, multiplier, &source_type, target_type)?;- }+ ensure_timestamp_in_bounds(value, multiplier, &source_type, target_type)?;}let scalar_array = self.to_array()?;@@ -5104,7 +5107,7 @@ mod tests {use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer};use arrow::compute::{is_null, kernels};use arrow::datatypes::{- ArrowNumericType, Fields, Float64Type, TimeUnit, DECIMAL256_MAX_PRECISION,+ ArrowNumericType, DECIMAL256_MAX_PRECISION, Fields, Float64Type, TimeUnit,};use arrow::error::ArrowError;use arrow::util::pretty::pretty_format_columns;@@ -5669,7 +5672,10 @@ mod tests {.sub_checked(&int_value_2).unwrap_err().strip_backtrace();- assert_eq!(err, "Arrow error: Arithmetic overflow: Overflow happened on: 9223372036854775807 - -9223372036854775808")+ assert_eq!(+ err,+ "Arrow error: Arithmetic overflow: Overflow happened on: 9223372036854775807 - -9223372036854775808"+ )}#[test]@@ -5817,12 +5823,16 @@ mod tests {assert_eq!(123i128, array_decimal.value(0));assert_eq!(123i128, array_decimal.value(9));// test eq array- assert!(decimal_value- .eq_array(&array, 1)- .expect("Failed to compare arrays"));- assert!(decimal_value- .eq_array(&array, 5)- .expect("Failed to compare arrays"));+ assert!(+ decimal_value+ .eq_array(&array, 1)+ .expect("Failed to compare arrays")+ );+ assert!(+ decimal_value+ .eq_array(&array, 5)+ .expect("Failed to compare arrays")+ );// test try from arrayassert_eq!(decimal_value,@@ -5867,18 +5877,24 @@ mod tests {assert_eq!(4, array.len());assert_eq!(DataType::Decimal128(10, 2), array.data_type().clone());- assert!(ScalarValue::try_new_decimal128(1, 10, 2)- .unwrap()- .eq_array(&array, 0)- .expect("Failed to compare arrays"));- assert!(ScalarValue::try_new_decimal128(2, 10, 2)- .unwrap()- .eq_array(&array, 1)- .expect("Failed to compare arrays"));- assert!(ScalarValue::try_new_decimal128(3, 10, 2)- .unwrap()- .eq_array(&array, 2)- .expect("Failed to compare arrays"));+ assert!(+ ScalarValue::try_new_decimal128(1, 10, 2)+ .unwrap()+ .eq_array(&array, 0)+ .expect("Failed to compare arrays")+ );+ assert!(+ ScalarValue::try_new_decimal128(2, 10, 2)+ .unwrap()+ .eq_array(&array, 1)+ .expect("Failed to compare arrays")+ );+ assert!(+ ScalarValue::try_new_decimal128(3, 10, 2)+ .unwrap()+ .eq_array(&array, 2)+ .expect("Failed to compare arrays")+ );assert_eq!(ScalarValue::Decimal128(None, 10, 2),ScalarValue::try_from_array(&array, 3).unwrap()@@ -6823,7 +6839,9 @@ mod tests {for other_index in 0..array.len() {if index != other_index {assert!(- !scalar.eq_array(&array, other_index).expect("Failed to compare arrays"),+ !scalar+ .eq_array(&array, other_index)+ .expect("Failed to compare arrays"),"Expected {scalar:?} to be NOT equal to {array:?} at index {other_index}");}diff --git a/datafusion/common/src/stats.rs b/datafusion/common/src/stats.rsindex da298c20eb..b42e98d130 100644--- a/datafusion/common/src/stats.rs+++ b/datafusion/common/src/stats.rs@@ -961,9 +961,11 @@ mod tests {Precision::Exact(ScalarValue::Int64(None)),);// Overflow returns error- assert!(Precision::Exact(ScalarValue::Int32(Some(256)))- .cast_to(&DataType::Int8)- .is_err());+ assert!(+ Precision::Exact(ScalarValue::Int32(Some(256)))+ .cast_to(&DataType::Int8)+ .is_err()+ );}#[test]@@ -1215,7 +1217,10 @@ mod tests {let items = vec![stats1, stats2];let e = Statistics::try_merge_iter(&items, &schema).unwrap_err();- assert_contains!(e.to_string(), "Error during planning: Cannot merge statistics with different number of columns: 0 vs 1");+ assert_contains!(+ e.to_string(),+ "Error during planning: Cannot merge statistics with different number of columns: 0 vs 1"+ );}#[test]diff --git a/datafusion/common/src/test_util.rs b/datafusion/common/src/test_util.rsindex c51dea1c4d..f060704944 100644--- a/datafusion/common/src/test_util.rs+++ b/datafusion/common/src/test_util.rs@@ -735,32 +735,34 @@ mod tests {let non_existing = cwd.join("non-existing-dir").display().to_string();let non_existing_str = non_existing.as_str();- env::set_var(udf_env, non_existing_str);- let res = get_data_dir(udf_env, existing_str);- assert!(res.is_err());-- env::set_var(udf_env, "");- let res = get_data_dir(udf_env, existing_str);- assert!(res.is_ok());- assert_eq!(res.unwrap(), existing_pb);-- env::set_var(udf_env, " ");- let res = get_data_dir(udf_env, existing_str);- assert!(res.is_ok());- assert_eq!(res.unwrap(), existing_pb);-- env::set_var(udf_env, existing_str);- let res = get_data_dir(udf_env, existing_str);- assert!(res.is_ok());- assert_eq!(res.unwrap(), existing_pb);-- env::remove_var(udf_env);- let res = get_data_dir(udf_env, non_existing_str);- assert!(res.is_err());-- let res = get_data_dir(udf_env, existing_str);- assert!(res.is_ok());- assert_eq!(res.unwrap(), existing_pb);+ unsafe {+ env::set_var(udf_env, non_existing_str);+ let res = get_data_dir(udf_env, existing_str);+ assert!(res.is_err());++ env::set_var(udf_env, "");+ let res = get_data_dir(udf_env, existing_str);+ assert!(res.is_ok());+ assert_eq!(res.unwrap(), existing_pb);++ env::set_var(udf_env, " ");+ let res = get_data_dir(udf_env, existing_str);+ assert!(res.is_ok());+ assert_eq!(res.unwrap(), existing_pb);++ env::set_var(udf_env, existing_str);+ let res = get_data_dir(udf_env, existing_str);+ assert!(res.is_ok());+ assert_eq!(res.unwrap(), existing_pb);++ env::remove_var(udf_env);+ let res = get_data_dir(udf_env, non_existing_str);+ assert!(res.is_err());++ let res = get_data_dir(udf_env, existing_str);+ assert!(res.is_ok());+ assert_eq!(res.unwrap(), existing_pb);+ }}#[test]diff --git a/datafusion/common/src/tree_node.rs b/datafusion/common/src/tree_node.rsindex 9b36266eec..1e7c02e424 100644--- a/datafusion/common/src/tree_node.rs+++ b/datafusion/common/src/tree_node.rs@@ -956,12 +956,12 @@ impl<'a, T: 'a, C0: TreeNodeContainer<'a, T>, C1: TreeNodeContainer<'a, T>>}impl<- 'a,- T: 'a,- C0: TreeNodeContainer<'a, T>,- C1: TreeNodeContainer<'a, T>,- C2: TreeNodeContainer<'a, T>,- > TreeNodeContainer<'a, T> for (C0, C1, C2)+ 'a,+ T: 'a,+ C0: TreeNodeContainer<'a, T>,+ C1: TreeNodeContainer<'a, T>,+ C2: TreeNodeContainer<'a, T>,+> TreeNodeContainer<'a, T> for (C0, C1, C2){fn apply_elements<F: FnMut(&'a T) -> Result<TreeNodeRecursion>>(&'a self,@@ -992,13 +992,13 @@ impl<}impl<- 'a,- T: 'a,- C0: TreeNodeContainer<'a, T>,- C1: TreeNodeContainer<'a, T>,- C2: TreeNodeContainer<'a, T>,- C3: TreeNodeContainer<'a, T>,- > TreeNodeContainer<'a, T> for (C0, C1, C2, C3)+ 'a,+ T: 'a,+ C0: TreeNodeContainer<'a, T>,+ C1: TreeNodeContainer<'a, T>,+ C2: TreeNodeContainer<'a, T>,+ C3: TreeNodeContainer<'a, T>,+> TreeNodeContainer<'a, T> for (C0, C1, C2, C3){fn apply_elements<F: FnMut(&'a T) -> Result<TreeNodeRecursion>>(&'a self,@@ -1090,12 +1090,12 @@ impl<'a, T: 'a, C0: TreeNodeContainer<'a, T>, C1: TreeNodeContainer<'a, T>>}impl<- 'a,- T: 'a,- C0: TreeNodeContainer<'a, T>,- C1: TreeNodeContainer<'a, T>,- C2: TreeNodeContainer<'a, T>,- > TreeNodeRefContainer<'a, T> for (&'a C0, &'a C1, &'a C2)+ 'a,+ T: 'a,+ C0: TreeNodeContainer<'a, T>,+ C1: TreeNodeContainer<'a, T>,+ C2: TreeNodeContainer<'a, T>,+> TreeNodeRefContainer<'a, T> for (&'a C0, &'a C1, &'a C2){fn apply_ref_elements<F: FnMut(&'a T) -> Result<TreeNodeRecursion>>(&self,@@ -1109,13 +1109,13 @@ impl<}impl<- 'a,- T: 'a,- C0: TreeNodeContainer<'a, T>,- C1: TreeNodeContainer<'a, T>,- C2: TreeNodeContainer<'a, T>,- C3: TreeNodeContainer<'a, T>,- > TreeNodeRefContainer<'a, T> for (&'a C0, &'a C1, &'a C2, &'a C3)+ 'a,+ T: 'a,+ C0: TreeNodeContainer<'a, T>,+ C1: TreeNodeContainer<'a, T>,+ C2: TreeNodeContainer<'a, T>,+ C3: TreeNodeContainer<'a, T>,+> TreeNodeRefContainer<'a, T> for (&'a C0, &'a C1, &'a C2, &'a C3){fn apply_ref_elements<F: FnMut(&'a T) -> Result<TreeNodeRecursion>>(&self,@@ -1336,11 +1336,11 @@ pub(crate) mod tests {use std::collections::HashMap;use std::fmt::Display;+ use crate::Result;use crate::tree_node::{Transformed, TreeNode, TreeNodeContainer, TreeNodeRecursion, TreeNodeRewriter,TreeNodeVisitor,};- use crate::Result;#[derive(Debug, Eq, Hash, PartialEq, Clone)]pub struct TestTreeNode<T> {diff --git a/datafusion/common/src/types/native.rs b/datafusion/common/src/types/native.rsindex 5e6fca040c..766c504416 100644--- a/datafusion/common/src/types/native.rs+++ b/datafusion/common/src/types/native.rs@@ -19,11 +19,11 @@ use super::{LogicalField, LogicalFieldRef, LogicalFields, LogicalType, LogicalUnionFields,TypeSignature,};-use crate::error::{Result, _internal_err};+use crate::error::{_internal_err, Result};use arrow::compute::can_cast_types;use arrow::datatypes::{- DataType, Field, FieldRef, Fields, IntervalUnit, TimeUnit, UnionFields,- DECIMAL128_MAX_PRECISION, DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION,+ DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION, DECIMAL128_MAX_PRECISION, DataType,+ Field, FieldRef, Fields, IntervalUnit, TimeUnit, UnionFields,};use std::{fmt::Display, sync::Arc};@@ -364,7 +364,7 @@ impl LogicalType for NativeType {"Unavailable default cast for native type {} from physical type {}",self,origin- )+ );}})}diff --git a/datafusion/common/src/utils/memory.rs b/datafusion/common/src/utils/memory.rsindex a56b940fab..028273ee98 100644--- a/datafusion/common/src/utils/memory.rs+++ b/datafusion/common/src/utils/memory.rs@@ -17,8 +17,8 @@//! This module provides a function to estimate the memory size of a HashTable prior to allocation-use crate::error::_exec_datafusion_err;use crate::Result;+use crate::error::_exec_datafusion_err;use std::mem::size_of;/// Estimates the memory size required for a hash table prior to allocation.diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rsindex 6e7396a7c5..93ab49530f 100644--- a/datafusion/common/src/utils/mod.rs+++ b/datafusion/common/src/utils/mod.rs@@ -26,16 +26,16 @@ use crate::assert_or_internal_err;use crate::error::{_exec_datafusion_err, _internal_datafusion_err};use crate::{DataFusionError, Result, ScalarValue};use arrow::array::{- cast::AsArray, Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray,- OffsetSizeTrait,+ Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray, OffsetSizeTrait,+ cast::AsArray,};use arrow::buffer::OffsetBuffer;-use arrow::compute::{partition, SortColumn, SortOptions};+use arrow::compute::{SortColumn, SortOptions, partition};use arrow::datatypes::{DataType, Field, SchemaRef};#[cfg(feature = "sql")]use sqlparser::{ast::Ident, dialect::GenericDialect, parser::Parser};use std::borrow::{Borrow, Cow};-use std::cmp::{min, Ordering};+use std::cmp::{Ordering, min};use std::collections::HashSet;use std::num::NonZero;use std::ops::Range;@@ -266,10 +266,10 @@ fn needs_quotes(s: &str) -> bool {let mut chars = s.chars();// first char can not be a number unless escaped- if let Some(first_char) = chars.next() {- if !(first_char.is_ascii_lowercase() || first_char == '_') {- return true;- }+ if let Some(first_char) = chars.next()+ && !(first_char.is_ascii_lowercase() || first_char == '_')+ {+ return true;}!chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_')@@ -943,8 +943,6 @@ mod tests {use super::*;use crate::ScalarValue::Null;use arrow::array::Float64Array;- use sqlparser::ast::Ident;- use sqlparser::tokenizer::Span;#[test]fn test_bisect_linear_left_and_right() -> Result<()> {@@ -1173,7 +1171,7 @@ mod tests {let expected_parsed = vec![Ident {value: identifier.to_string(),quote_style,- span: Span::empty(),+ span: sqlparser::tokenizer::Span::empty(),}];assert_eq!(diff --git a/datafusion/doc/Cargo.toml b/datafusion/doc/Cargo.tomlindex c1368c1531..94edf70bc7 100644--- a/datafusion/doc/Cargo.toml+++ b/datafusion/doc/Cargo.toml@@ -21,7 +21,7 @@ description = "Documentation module for DataFusion query engine"keywords = ["datafusion", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = { workspace = true }+edition = "2024"homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/doc/src/udf.rs b/datafusion/doc/src/udf.rsindex 3d18c9ac27..d1f51d9194 100644--- a/datafusion/doc/src/udf.rs+++ b/datafusion/doc/src/udf.rs@@ -127,6 +127,8 @@ The following regular expression functions are supported:"#,pub const DOC_SECTION_UNION: DocSection = DocSection {include: true,label: "Union Functions",- description: Some("Functions to work with the union data type, also know as tagged unions, variant types, enums or sum types. Note: Not related to the SQL UNION operator"),+ description: Some(+ "Functions to work with the union data type, also know as tagged unions, variant types, enums or sum types. Note: Not related to the SQL UNION operator",+ ),};}diff --git a/datafusion/expr-common/Cargo.toml b/datafusion/expr-common/Cargo.tomlindex 5ee46b454e..7b13334a6a 100644--- a/datafusion/expr-common/Cargo.toml+++ b/datafusion/expr-common/Cargo.toml@@ -21,7 +21,7 @@ description = "Logical plan and expression representation for DataFusion query ekeywords = ["datafusion", "logical", "plan", "expressions"]readme = "README.md"version = { workspace = true }-edition = { workspace = true }+edition = "2024"homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/expr-common/src/accumulator.rs b/datafusion/expr-common/src/accumulator.rsindex 2829a9416f..fc4e90114b 100644--- a/datafusion/expr-common/src/accumulator.rs+++ b/datafusion/expr-common/src/accumulator.rs@@ -18,7 +18,7 @@//! Accumulator module contains the trait definition for aggregation function's accumulators.use arrow::array::ArrayRef;-use datafusion_common::{internal_err, Result, ScalarValue};+use datafusion_common::{Result, ScalarValue, internal_err};use std::fmt::Debug;/// Tracks an aggregate function's state.diff --git a/datafusion/expr-common/src/casts.rs b/datafusion/expr-common/src/casts.rsindex 8939ff1371..dc0bd74b1f 100644--- a/datafusion/expr-common/src/casts.rs+++ b/datafusion/expr-common/src/casts.rs@@ -24,10 +24,9 @@use std::cmp::Ordering;use arrow::datatypes::{- DataType, TimeUnit, MAX_DECIMAL128_FOR_EACH_PRECISION,- MAX_DECIMAL32_FOR_EACH_PRECISION, MAX_DECIMAL64_FOR_EACH_PRECISION,- MIN_DECIMAL128_FOR_EACH_PRECISION, MIN_DECIMAL32_FOR_EACH_PRECISION,- MIN_DECIMAL64_FOR_EACH_PRECISION,+ DataType, MAX_DECIMAL32_FOR_EACH_PRECISION, MAX_DECIMAL64_FOR_EACH_PRECISION,+ MAX_DECIMAL128_FOR_EACH_PRECISION, MIN_DECIMAL32_FOR_EACH_PRECISION,+ MIN_DECIMAL64_FOR_EACH_PRECISION, MIN_DECIMAL128_FOR_EACH_PRECISION, TimeUnit,};use arrow::temporal_conversions::{MICROSECONDS, MILLISECONDS, NANOSECONDS};use datafusion_common::ScalarValue;@@ -382,7 +381,7 @@ fn try_cast_binary(#[cfg(test)]mod tests {use super::*;- use arrow::compute::{cast_with_options, CastOptions};+ use arrow::compute::{CastOptions, cast_with_options};use arrow::datatypes::{Field, Fields, TimeUnit};use std::sync::Arc;diff --git a/datafusion/expr-common/src/columnar_value.rs b/datafusion/expr-common/src/columnar_value.rsindex 585b47a980..99c21d4abd 100644--- a/datafusion/expr-common/src/columnar_value.rs+++ b/datafusion/expr-common/src/columnar_value.rs@@ -19,16 +19,16 @@use arrow::{array::{Array, ArrayRef, Date32Array, Date64Array, NullArray},- compute::{kernels, max, min, CastOptions},+ compute::{CastOptions, kernels, max, min},datatypes::DataType,util::pretty::pretty_format_columns,};use datafusion_common::internal_datafusion_err;use datafusion_common::{+ Result, ScalarValue,format::DEFAULT_CAST_OPTIONS,internal_err,scalar::{date_to_timestamp_multiplier, ensure_timestamp_in_bounds},- Result, ScalarValue,};use std::fmt;use std::sync::Arc;diff --git a/datafusion/expr-common/src/groups_accumulator.rs b/datafusion/expr-common/src/groups_accumulator.rsindex 9bcc1edff8..860e69245a 100644--- a/datafusion/expr-common/src/groups_accumulator.rs+++ b/datafusion/expr-common/src/groups_accumulator.rs@@ -18,7 +18,7 @@//! Vectorized [`GroupsAccumulator`]use arrow::array::{ArrayRef, BooleanArray};-use datafusion_common::{not_impl_err, Result};+use datafusion_common::{Result, not_impl_err};/// Describes how many rows should be emitted during grouping.#[derive(Debug, Clone, Copy, PartialEq, Eq)]diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rsindex b9f8102f34..a43fe8c415 100644--- a/datafusion/expr-common/src/interval_arithmetic.rs+++ b/datafusion/expr-common/src/interval_arithmetic.rs@@ -22,18 +22,18 @@ use std::fmt::{self, Display, Formatter};use std::ops::{AddAssign, SubAssign};use crate::operator::Operator;-use crate::type_coercion::binary::{comparison_coercion_numeric, BinaryTypeCoercer};+use crate::type_coercion::binary::{BinaryTypeCoercer, comparison_coercion_numeric};-use arrow::compute::{cast_with_options, CastOptions};+use arrow::compute::{CastOptions, cast_with_options};use arrow::datatypes::{- DataType, IntervalDayTime, IntervalMonthDayNano, IntervalUnit, TimeUnit,+ DataType, IntervalDayTime, IntervalMonthDayNano, IntervalUnit,MAX_DECIMAL128_FOR_EACH_PRECISION, MAX_DECIMAL256_FOR_EACH_PRECISION,- MIN_DECIMAL128_FOR_EACH_PRECISION, MIN_DECIMAL256_FOR_EACH_PRECISION,+ MIN_DECIMAL128_FOR_EACH_PRECISION, MIN_DECIMAL256_FOR_EACH_PRECISION, TimeUnit,};use datafusion_common::rounding::{alter_fp_rounding_mode, next_down, next_up};use datafusion_common::{- assert_eq_or_internal_err, assert_or_internal_err, internal_err, DataFusionError,- Result, ScalarValue,+ DataFusionError, Result, ScalarValue, assert_eq_or_internal_err,+ assert_or_internal_err, internal_err,};macro_rules! get_extreme_value {@@ -2202,7 +2202,7 @@ impl NullableInterval {mod tests {use crate::{interval_arithmetic::{- handle_overflow, next_value, prev_value, satisfy_greater, Interval,+ Interval, handle_overflow, next_value, prev_value, satisfy_greater,},operator::Operator,};@@ -2248,10 +2248,12 @@ mod tests {ScalarValue::Float64(Some(1e-6)),];values.into_iter().zip(eps).for_each(|(value, eps)| {- assert!(next_value(value.clone())- .sub(value.clone())- .unwrap()- .lt(&eps));+ assert!(+ next_value(value.clone())+ .sub(value.clone())+ .unwrap()+ .lt(&eps)+ );assert!(value.sub(prev_value(value.clone())).unwrap().lt(&eps));assert_ne!(next_value(value.clone()), value);assert_ne!(prev_value(value.clone()), value);@@ -2841,18 +2843,26 @@ mod tests {// not contain `null`.#[test]fn test_uncertain_boolean_interval() {- assert!(Interval::TRUE_OR_FALSE- .contains_value(ScalarValue::Boolean(Some(true)))- .unwrap());- assert!(Interval::TRUE_OR_FALSE- .contains_value(ScalarValue::Boolean(Some(false)))- .unwrap());- assert!(!Interval::TRUE_OR_FALSE- .contains_value(ScalarValue::Boolean(None))- .unwrap());- assert!(!Interval::TRUE_OR_FALSE- .contains_value(ScalarValue::Null)- .unwrap());+ assert!(+ Interval::TRUE_OR_FALSE+ .contains_value(ScalarValue::Boolean(Some(true)))+ .unwrap()+ );+ assert!(+ Interval::TRUE_OR_FALSE+ .contains_value(ScalarValue::Boolean(Some(false)))+ .unwrap()+ );+ assert!(+ !Interval::TRUE_OR_FALSE+ .contains_value(ScalarValue::Boolean(None))+ .unwrap()+ );+ assert!(+ !Interval::TRUE_OR_FALSE+ .contains_value(ScalarValue::Null)+ .unwrap()+ );}#[test]diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rsindex 6ee1c4a2a4..0c20bf0e86 100644--- a/datafusion/expr-common/src/signature.rs+++ b/datafusion/expr-common/src/signature.rs@@ -24,7 +24,7 @@ use crate::type_coercion::aggregates::NUMERICS;use arrow::datatypes::{DataType, Decimal128Type, DecimalType, IntervalUnit, TimeUnit};use datafusion_common::types::{LogicalType, LogicalTypeRef, NativeType};use datafusion_common::utils::ListCoercion;-use datafusion_common::{internal_err, plan_err, Result};+use datafusion_common::{Result, internal_err, plan_err};use indexmap::IndexSet;use itertools::Itertools;@@ -558,9 +558,11 @@ impl TypeSignature {vec![Self::join_types(types, ", ")]}TypeSignature::Any(arg_count) => {- vec![std::iter::repeat_n("Any", *arg_count)- .collect::<Vec<&str>>()- .join(", ")]+ vec![+ std::iter::repeat_n("Any", *arg_count)+ .collect::<Vec<&str>>()+ .join(", "),+ ]}TypeSignature::UserDefined => {vec!["UserDefined".to_string()]@@ -607,87 +609,103 @@ impl TypeSignature {match self {TypeSignature::Exact(types) => {if let Some(names) = parameter_names {- vec![names- .iter()- .zip(types.iter())- .map(|(name, typ)| format!("{name}: {typ}"))- .collect::<Vec<_>>()- .join(", ")]+ vec![+ names+ .iter()+ .zip(types.iter())+ .map(|(name, typ)| format!("{name}: {typ}"))+ .collect::<Vec<_>>()+ .join(", "),+ ]} else {vec![Self::join_types(types, ", ")]}}TypeSignature::Any(count) => {if let Some(names) = parameter_names {- vec![names- .iter()- .take(*count)- .map(|name| format!("{name}: Any"))- .collect::<Vec<_>>()- .join(", ")]+ vec![+ names+ .iter()+ .take(*count)+ .map(|name| format!("{name}: Any"))+ .collect::<Vec<_>>()+ .join(", "),+ ]} else {- vec![std::iter::repeat_n("Any", *count)- .collect::<Vec<&str>>()- .join(", ")]+ vec![+ std::iter::repeat_n("Any", *count)+ .collect::<Vec<&str>>()+ .join(", "),+ ]}}TypeSignature::Uniform(count, types) => {if let Some(names) = parameter_names {let type_str = Self::join_types(types, "/");- vec![names- .iter()- .take(*count)- .map(|name| format!("{name}: {type_str}"))- .collect::<Vec<_>>()- .join(", ")]+ vec![+ names+ .iter()+ .take(*count)+ .map(|name| format!("{name}: {type_str}"))+ .collect::<Vec<_>>()+ .join(", "),+ ]} else {self.to_string_repr()}}TypeSignature::Coercible(coercions) => {if let Some(names) = parameter_names {- vec![names- .iter()- .zip(coercions.iter())- .map(|(name, coercion)| format!("{name}: {coercion}"))- .collect::<Vec<_>>()- .join(", ")]+ vec![+ names+ .iter()+ .zip(coercions.iter())+ .map(|(name, coercion)| format!("{name}: {coercion}"))+ .collect::<Vec<_>>()+ .join(", "),+ ]} else {vec![Self::join_types(coercions, ", ")]}}TypeSignature::Comparable(count) => {if let Some(names) = parameter_names {- vec![names- .iter()- .take(*count)- .map(|name| format!("{name}: Comparable"))- .collect::<Vec<_>>()- .join(", ")]+ vec![+ names+ .iter()+ .take(*count)+ .map(|name| format!("{name}: Comparable"))+ .collect::<Vec<_>>()+ .join(", "),+ ]} else {self.to_string_repr()}}TypeSignature::Numeric(count) => {if let Some(names) = parameter_names {- vec![names- .iter()- .take(*count)- .map(|name| format!("{name}: Numeric"))- .collect::<Vec<_>>()- .join(", ")]+ vec![+ names+ .iter()+ .take(*count)+ .map(|name| format!("{name}: Numeric"))+ .collect::<Vec<_>>()+ .join(", "),+ ]} else {self.to_string_repr()}}TypeSignature::String(count) => {if let Some(names) = parameter_names {- vec![names- .iter()- .take(*count)- .map(|name| format!("{name}: String"))- .collect::<Vec<_>>()- .join(", ")]+ vec![+ names+ .iter()+ .take(*count)+ .map(|name| format!("{name}: String"))+ .collect::<Vec<_>>()+ .join(", "),+ ]} else {self.to_string_repr()}@@ -697,28 +715,34 @@ impl TypeSignature {if let Some(names) = parameter_names {match array_sig {ArrayFunctionSignature::Array { arguments, .. } => {- vec![names- .iter()- .zip(arguments.iter())- .map(|(name, arg_type)| format!("{name}: {arg_type}"))- .collect::<Vec<_>>()- .join(", ")]+ vec![+ names+ .iter()+ .zip(arguments.iter())+ .map(|(name, arg_type)| format!("{name}: {arg_type}"))+ .collect::<Vec<_>>()+ .join(", "),+ ]}ArrayFunctionSignature::RecursiveArray => {- vec![names- .iter()- .take(1)- .map(|name| format!("{name}: recursive_array"))- .collect::<Vec<_>>()- .join(", ")]+ vec![+ names+ .iter()+ .take(1)+ .map(|name| format!("{name}: recursive_array"))+ .collect::<Vec<_>>()+ .join(", "),+ ]}ArrayFunctionSignature::MapArray => {- vec![names- .iter()- .take(1)- .map(|name| format!("{name}: map_array"))- .collect::<Vec<_>>()- .join(", ")]+ vec![+ names+ .iter()+ .take(1)+ .map(|name| format!("{name}: map_array"))+ .collect::<Vec<_>>()+ .join(", "),+ ]}}} else {@@ -1538,10 +1562,12 @@ mod tests {.with_parameter_names(vec!["count".to_string()]); // Only 1 name for 2 argsassert!(result.is_err());- assert!(result- .unwrap_err()- .to_string()- .contains("does not match signature arity"));+ assert!(+ result+ .unwrap_err()+ .to_string()+ .contains("does not match signature arity")+ );}#[test]@@ -1553,10 +1579,12 @@ mod tests {.with_parameter_names(vec!["count".to_string(), "count".to_string()]);assert!(result.is_err());- assert!(result- .unwrap_err()- .to_string()- .contains("Duplicate parameter name"));+ assert!(+ result+ .unwrap_err()+ .to_string()+ .contains("Duplicate parameter name")+ );}#[test]@@ -1565,10 +1593,12 @@ mod tests {.with_parameter_names(vec!["arg".to_string()]);assert!(result.is_err());- assert!(result- .unwrap_err()- .to_string()- .contains("variable arity signature"));+ assert!(+ result+ .unwrap_err()+ .to_string()+ .contains("variable arity signature")+ );}#[test]diff --git a/datafusion/expr-common/src/statistics.rs b/datafusion/expr-common/src/statistics.rsindex 7961bf0872..72da2caa93 100644--- a/datafusion/expr-common/src/statistics.rs+++ b/datafusion/expr-common/src/statistics.rs@@ -17,7 +17,7 @@use std::f64::consts::LN_2;-use crate::interval_arithmetic::{apply_operator, Interval};+use crate::interval_arithmetic::{Interval, apply_operator};use crate::operator::Operator;use crate::type_coercion::binary::binary_numeric_coercion;@@ -25,8 +25,8 @@ use arrow::array::ArrowNativeTypeOp;use arrow::datatypes::DataType;use datafusion_common::rounding::alter_fp_rounding_mode;use datafusion_common::{- assert_eq_or_internal_err, assert_ne_or_internal_err, assert_or_internal_err,- internal_err, not_impl_err, DataFusionError, Result, ScalarValue,+ DataFusionError, Result, ScalarValue, assert_eq_or_internal_err,+ assert_ne_or_internal_err, assert_or_internal_err, internal_err, not_impl_err,};/// This object defines probabilistic distributions that encode uncertain@@ -878,11 +878,11 @@ pub fn compute_variance(#[cfg(test)]mod tests {use super::{+ BernoulliDistribution, Distribution, GaussianDistribution, UniformDistribution,combine_bernoullis, combine_gaussians, compute_mean, compute_median,compute_variance, create_bernoulli_from_comparison, new_generic_from_binary_op,- BernoulliDistribution, Distribution, GaussianDistribution, UniformDistribution,};- use crate::interval_arithmetic::{apply_operator, Interval};+ use crate::interval_arithmetic::{Interval, apply_operator};use crate::operator::Operator;use arrow::datatypes::DataType;diff --git a/datafusion/expr-common/src/type_coercion/aggregates.rs b/datafusion/expr-common/src/type_coercion/aggregates.rsindex 55a8843394..01d093950d 100644--- a/datafusion/expr-common/src/type_coercion/aggregates.rs+++ b/datafusion/expr-common/src/type_coercion/aggregates.rs@@ -18,7 +18,7 @@use crate::signature::TypeSignature;use arrow::datatypes::{DataType, FieldRef};-use datafusion_common::{internal_err, plan_err, Result};+use datafusion_common::{Result, internal_err, plan_err};// TODO: remove usage of these (INTEGERS and NUMERICS) in favour of signatures// see https://github.com/apache/datafusion/issues/18092diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rsindex 4aacc7533c..18603991ae 100644--- a/datafusion/expr-common/src/type_coercion/binary.rs+++ b/datafusion/expr-common/src/type_coercion/binary.rs@@ -22,18 +22,18 @@ use std::sync::Arc;use crate::operator::Operator;-use arrow::array::{new_empty_array, Array};+use arrow::array::{Array, new_empty_array};use arrow::compute::can_cast_types;use arrow::datatypes::{- DataType, Field, FieldRef, Fields, TimeUnit, DECIMAL128_MAX_PRECISION,- DECIMAL128_MAX_SCALE, DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE,DECIMAL32_MAX_PRECISION, DECIMAL32_MAX_SCALE, DECIMAL64_MAX_PRECISION,- DECIMAL64_MAX_SCALE,+ DECIMAL64_MAX_SCALE, DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE,+ DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE, DataType, Field, FieldRef, Fields,+ TimeUnit,};use datafusion_common::types::NativeType;use datafusion_common::{- exec_err, internal_err, not_impl_err, plan_datafusion_err, plan_err, Diagnostic,- Result, Span, Spans,+ Diagnostic, Result, Span, Spans, exec_err, internal_err, not_impl_err,+ plan_datafusion_err, plan_err,};use itertools::Itertools;@@ -184,8 +184,8 @@ impl<'a> BinaryTypeCoercer<'a> {}fn signature_inner(&'a self, lhs: &DataType, rhs: &DataType) -> Result<Signature> {- use arrow::datatypes::DataType::*;use Operator::*;+ use arrow::datatypes::DataType::*;let result = match self.op {Eq |NotEq |@@ -751,7 +751,11 @@ pub fn try_type_union_resolution_with_struct(let keys = fields.iter().map(|f| f.name().to_owned()).join(",");if let Some(ref k) = keys_string {if *k != keys {- return exec_err!("Expect same keys for struct type but got mismatched pair {} and {}", *k, keys);+ return exec_err!(+ "Expect same keys for struct type but got mismatched pair {} and {}",+ *k,+ keys+ );}} else {keys_string = Some(keys);@@ -765,7 +769,9 @@ pub fn try_type_union_resolution_with_struct({fields.iter().map(|f| f.data_type().to_owned()).collect()} else {- return internal_err!("Struct type is checked is the previous function, so this should be unreachable");+ return internal_err!(+ "Struct type is checked is the previous function, so this should be unreachable"+ );};for data_type in data_types.iter().skip(1) {diff --git a/datafusion/expr-common/src/type_coercion/binary/tests/arithmetic.rs b/datafusion/expr-common/src/type_coercion/binary/tests/arithmetic.rsindex 63945a4dab..4a3ccf7bb7 100644--- a/datafusion/expr-common/src/type_coercion/binary/tests/arithmetic.rs+++ b/datafusion/expr-common/src/type_coercion/binary/tests/arithmetic.rs@@ -25,7 +25,10 @@ fn test_coercion_error() -> Result<()> {let result_type = coercer.get_input_types();let e = result_type.unwrap_err();- assert_eq!(e.strip_backtrace(), "Error during planning: Cannot coerce arithmetic expression Float32 + Utf8 to valid types");+ assert_eq!(+ e.strip_backtrace(),+ "Error during planning: Cannot coerce arithmetic expression Float32 + Utf8 to valid types"+ );Ok(())}@@ -146,14 +149,18 @@ fn test_type_coercion_arithmetic() -> Result<()> {// (_, Float32) | (Float32, _) => Some(Float32)test_coercion_binary_rule_multiple!(Float32,- [Float32, Float16, Int64, UInt64, Int32, UInt32, Int16, UInt16, Int8, UInt8],+ [+ Float32, Float16, Int64, UInt64, Int32, UInt32, Int16, UInt16, Int8, UInt8+ ],Operator::Plus,Float32);// (_, Float16) | (Float16, _) => Some(Float16)test_coercion_binary_rule_multiple!(Float16,- [Float16, Int64, UInt64, Int32, UInt32, Int16, UInt16, Int8, UInt8],+ [+ Float16, Int64, UInt64, Int32, UInt32, Int16, UInt16, Int8, UInt8+ ],Operator::Plus,Float16);diff --git a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rsindex 5401264e43..d19b2c7ba9 100644--- a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs+++ b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs@@ -778,8 +778,14 @@ fn test_decimal_cross_variant_comparison_coercion() -> Result<()> {for op in comparison_op_types {let (lhs, rhs) =BinaryTypeCoercer::new(&lhs_type, &op, &rhs_type).get_input_types()?;- assert_eq!(expected_type, lhs, "Coercion of type {lhs_type:?} with {rhs_type:?} resulted in unexpected type: {lhs:?}");- assert_eq!(expected_type, rhs, "Coercion of type {rhs_type:?} with {lhs_type:?} resulted in unexpected type: {rhs:?}");+ assert_eq!(+ expected_type, lhs,+ "Coercion of type {lhs_type:?} with {rhs_type:?} resulted in unexpected type: {lhs:?}"+ );+ assert_eq!(+ expected_type, rhs,+ "Coercion of type {rhs_type:?} with {lhs_type:?} resulted in unexpected type: {rhs:?}"+ );}}diff --git a/datafusion/macros/Cargo.toml b/datafusion/macros/Cargo.tomlindex 35714bfe96..ba7368b0fe 100644--- a/datafusion/macros/Cargo.toml+++ b/datafusion/macros/Cargo.toml@@ -21,7 +21,7 @@ description = "Procedural macros for DataFusion query engine"keywords = ["datafusion", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = { workspace = true }+edition = "2024"homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/macros/src/user_doc.rs b/datafusion/macros/src/user_doc.rsindex 58c2cc2b1b..ce9e7d55ef 100644--- a/datafusion/macros/src/user_doc.rs+++ b/datafusion/macros/src/user_doc.rs@@ -25,7 +25,7 @@ extern crate proc_macro;use datafusion_doc::scalar_doc_sections::doc_sections_const;use proc_macro::TokenStream;use quote::quote;-use syn::{parse_macro_input, DeriveInput, LitStr};+use syn::{DeriveInput, LitStr, parse_macro_input};/// This procedural macro is intended to parse a rust custom attribute and create user documentation/// from it by constructing a `DocumentBuilder()` automatically. The `Documentation` can bediff --git a/datafusion/proto-common/Cargo.toml b/datafusion/proto-common/Cargo.tomlindex c67c8892a3..f92ee554c4 100644--- a/datafusion/proto-common/Cargo.toml+++ b/datafusion/proto-common/Cargo.toml@@ -21,7 +21,7 @@ description = "Protobuf serialization of DataFusion common types"keywords = ["arrow", "query", "sql"]readme = "README.md"version = { workspace = true }-edition = { workspace = true }+edition = "2024"homepage = { workspace = true }repository = { workspace = true }license = { workspace = true }diff --git a/datafusion/proto-common/src/common.rs b/datafusion/proto-common/src/common.rsindex 9af63e3b07..d5046aee2e 100644--- a/datafusion/proto-common/src/common.rs+++ b/datafusion/proto-common/src/common.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use datafusion_common::{internal_datafusion_err, DataFusionError};+use datafusion_common::{DataFusionError, internal_datafusion_err};/// Return a `DataFusionError::Internal` with the given messagepub fn proto_error<S: Into<String>>(message: S) -> DataFusionError {diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex 2d65a55d37..1a38ed4363 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -25,12 +25,14 @@ use arrow::array::{ArrayRef, AsArray};use arrow::buffer::Buffer;use arrow::csv::WriterBuilder;use arrow::datatypes::{- i256, DataType, Field, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit,- Schema, TimeUnit, UnionFields, UnionMode,+ DataType, Field, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, Schema,+ TimeUnit, UnionFields, UnionMode, i256,};use arrow::ipc::{reader::read_record_batch, root_as_message};use datafusion_common::{+ Column, ColumnStatistics, Constraint, Constraints, DFSchema, DFSchemaRef,+ DataFusionError, JoinSide, ScalarValue, Statistics, TableReference,arrow_datafusion_err,config::{CsvOptions, JsonOptions, ParquetColumnOptions, ParquetOptions,@@ -40,8 +42,6 @@ use datafusion_common::{parsers::CompressionTypeVariant,plan_datafusion_err,stats::Precision,- Column, ColumnStatistics, Constraint, Constraints, DFSchema, DFSchemaRef,- DataFusionError, JoinSide, ScalarValue, Statistics, TableReference,};#[derive(Debug)]diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rsindex e9de1d9e9a..0152d57832 100644--- a/datafusion/proto-common/src/to_proto/mod.rs+++ b/datafusion/proto-common/src/to_proto/mod.rs@@ -20,7 +20,7 @@ use std::sync::Arc;use crate::protobuf_common as protobuf;use crate::protobuf_common::{- arrow_type::ArrowTypeEnum, scalar_value::Value, EmptyMessage,+ EmptyMessage, arrow_type::ArrowTypeEnum, scalar_value::Value,};use arrow::array::{ArrayRef, RecordBatch};use arrow::csv::WriterBuilder;@@ -32,6 +32,8 @@ use arrow::ipc::writer::{CompressionContext, DictionaryTracker, IpcDataGenerator, IpcWriteOptions,};use datafusion_common::{+ Column, ColumnStatistics, Constraint, Constraints, DFSchema, DFSchemaRef,+ DataFusionError, JoinSide, ScalarValue, Statistics,config::{CsvOptions, JsonOptions, ParquetColumnOptions, ParquetOptions,TableParquetOptions,@@ -40,8 +42,6 @@ use datafusion_common::{parsers::CompressionTypeVariant,plan_datafusion_err,stats::Precision,- Column, ColumnStatistics, Constraint, Constraints, DFSchema, DFSchemaRef,- DataFusionError, JoinSide, ScalarValue, Statistics,};#[derive(Debug)]@@ -1018,11 +1018,11 @@ fn encode_scalar_nested_value())})?;- let gen = IpcDataGenerator {};+ let ipc_gen = IpcDataGenerator {};let mut dict_tracker = DictionaryTracker::new(false);let write_options = IpcWriteOptions::default();let mut compression_context = CompressionContext::default();- let (encoded_dictionaries, encoded_message) = gen+ let (encoded_dictionaries, encoded_message) = ipc_gen.encode(&batch,&mut dict_tracker,diff --git a/dev/depcheck/Cargo.toml b/dev/depcheck/Cargo.tomlindex 23cefaec43..3e4bf39cce 100644--- a/dev/depcheck/Cargo.toml+++ b/dev/depcheck/Cargo.toml@@ -18,8 +18,9 @@# Circular dependency checker for DataFusion[package]name = "depcheck"+edition = "2024"# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html[dependencies]-cargo = "0.81.0"+cargo = "0.92.0"diff --git a/dev/depcheck/rust-toolchain.toml b/dev/depcheck/rust-toolchain.tomlnew file mode 100644index 0000000000..55d572362d--- /dev/null+++ b/dev/depcheck/rust-toolchain.toml@@ -0,0 +1,23 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# This file specifies the default version of Rust used+# to compile this workspace and run CI jobs.++[toolchain]+channel = "1.89.0"+components = ["rustfmt", "clippy"]diff --git a/dev/depcheck/src/main.rs b/dev/depcheck/src/main.rsindex 80feefcd1b..ebd79faa6f 100644--- a/dev/depcheck/src/main.rs+++ b/dev/depcheck/src/main.rs@@ -48,7 +48,7 @@ fn main() -> CargoResult<()> {root_cargo_toml.display());let workspace = cargo::core::Workspace::new(&root_cargo_toml, &gctx)?;- let (_, resolve) = cargo::ops::resolve_ws(&workspace)?;+ let (_, resolve) = cargo::ops::resolve_ws(&workspace, false)?;let mut package_deps = HashMap::new();for package_id in resolve