Apache DataFusion SQL Query Engine
Upgrade hashbrown to 0.16 (#19554)
Daniël Heresa6fd5cc840d1
diff --git a/Cargo.lock b/Cargo.lockindex 077efb0fae..1675f26e8a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2013,7 +2013,7 @@ dependencies = ["chrono","criterion","half",- "hashbrown 0.14.5",+ "hashbrown 0.16.1","hex","indexmap 2.12.1","insta",@@ -2495,7 +2495,7 @@ dependencies = ["datafusion-functions-aggregate-common","datafusion-physical-expr-common","half",- "hashbrown 0.14.5",+ "hashbrown 0.16.1","indexmap 2.12.1","insta","itertools 0.14.0",@@ -2530,7 +2530,8 @@ dependencies = ["chrono","datafusion-common","datafusion-expr-common",- "hashbrown 0.14.5",+ "hashbrown 0.16.1",+ "indexmap 2.12.1","itertools 0.14.0","parking_lot",]@@ -2578,7 +2579,7 @@ dependencies = ["datafusion-physical-expr-common","futures","half",- "hashbrown 0.14.5",+ "hashbrown 0.16.1","indexmap 2.12.1","insta","itertools 0.14.0",@@ -3120,6 +3121,12 @@ version = "0.1.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"+[[package]]+name = "foldhash"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"+[[package]]name = "form_urlencoded"version = "1.2.2"@@ -3371,10 +3378,6 @@ name = "hashbrown"version = "0.14.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"-dependencies = [- "ahash 0.8.12",- "allocator-api2",-][[package]]name = "hashbrown"@@ -3384,7 +3387,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"dependencies = ["allocator-api2","equivalent",- "foldhash",+ "foldhash 0.1.5",][[package]]@@ -3392,6 +3395,11 @@ name = "hashbrown"version = "0.16.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"+dependencies = [+ "allocator-api2",+ "equivalent",+ "foldhash 0.2.0",+][[package]]name = "heck"diff --git a/Cargo.toml b/Cargo.tomlindex 10fc88b705..d801bb6114 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -156,7 +156,7 @@ flate2 = "1.1.5"futures = "0.3"glob = "0.3.0"half = { version = "2.7.0", default-features = false }-hashbrown = { version = "0.14.5", features = ["raw"] }+hashbrown = { version = "0.16.1" }hex = { version = "0.4.3" }indexmap = "2.12.1"insta = { version = "1.45.0", features = ["glob", "filters"] }diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rsindex 3bec9bd35c..2dbc086886 100644--- a/datafusion/common/src/lib.rs+++ b/datafusion/common/src/lib.rs@@ -85,7 +85,7 @@ pub use functional_dependencies::{aggregate_functional_dependencies, get_required_group_by_exprs_indices,get_target_functional_dependencies,};-use hashbrown::hash_map::DefaultHashBuilder;+use hashbrown::DefaultHashBuilder;pub use join_type::{JoinConstraint, JoinSide, JoinType};pub use nested_struct::cast_column;pub use null_equality::NullEquality;diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rsindex 8491062124..9b2e7429ab 100644--- a/datafusion/functions-aggregate/src/array_agg.rs+++ b/datafusion/functions-aggregate/src/array_agg.rs@@ -1111,7 +1111,7 @@ mod tests {])])?;// without compaction, the size is 17112- assert_eq!(acc.size(), 2184);+ assert_eq!(acc.size(), 2224);Ok(())}diff --git a/datafusion/physical-expr-common/Cargo.toml b/datafusion/physical-expr-common/Cargo.tomlindex d292da212e..a81eafe196 100644--- a/datafusion/physical-expr-common/Cargo.toml+++ b/datafusion/physical-expr-common/Cargo.toml@@ -47,5 +47,6 @@ chrono = { workspace = true }datafusion-common = { workspace = true }datafusion-expr-common = { workspace = true }hashbrown = { workspace = true }+indexmap = { workspace = true }itertools = { workspace = true }parking_lot = { workspace = true }diff --git a/datafusion/physical-expr-common/src/sort_expr.rs b/datafusion/physical-expr-common/src/sort_expr.rsindex db30dd6ed2..fa961981c0 100644--- a/datafusion/physical-expr-common/src/sort_expr.rs+++ b/datafusion/physical-expr-common/src/sort_expr.rs@@ -31,7 +31,7 @@ use arrow::datatypes::Schema;use arrow::record_batch::RecordBatch;use datafusion_common::{HashSet, Result};use datafusion_expr_common::columnar_value::ColumnarValue;-+use indexmap::IndexSet;/// Represents Sort operation for a column in a RecordBatch////// Example:@@ -353,14 +353,14 @@ impl From<PhysicalSortRequirement> for PhysicalSortExpr {/// 1. It is non-degenerate, meaning it contains at least one element./// 2. It is duplicate-free, meaning it does not contain multiple entries for/// the same column.-#[derive(Debug, Clone)]+#[derive(Clone, Debug)]pub struct LexOrdering {/// Vector of sort expressions representing the lexicographical ordering.exprs: Vec<PhysicalSortExpr>,/// Set of expressions in the lexicographical ordering, used to ensure/// that the ordering is duplicate-free. Note that the elements in this/// set are the same underlying physical expressions as in `exprs`.- set: HashSet<Arc<dyn PhysicalExpr>>,+ set: IndexSet<Arc<dyn PhysicalExpr>>,}impl LexOrdering {@@ -371,7 +371,7 @@ impl LexOrdering {let mut candidate = Self {// not valid yet; valid publicly-returned instance must be non-emptyexprs: Vec::new(),- set: HashSet::new(),+ set: IndexSet::new(),};for expr in exprs {candidate.push(expr);@@ -421,7 +421,7 @@ impl LexOrdering {return false;}for PhysicalSortExpr { expr, .. } in self.exprs[len..].iter() {- self.set.remove(expr);+ self.set.swap_remove(expr);}self.exprs.truncate(len);truediff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rsindex 91d339910b..78478fc13e 100644--- a/datafusion/physical-expr/src/equivalence/class.rs+++ b/datafusion/physical-expr/src/equivalence/class.rs@@ -27,7 +27,7 @@ use crate::projection::ProjectionTargets;use crate::{PhysicalExpr, PhysicalExprRef, PhysicalSortExpr, PhysicalSortRequirement};use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};-use datafusion_common::{HashMap, JoinType, Result, ScalarValue};+use datafusion_common::{JoinType, Result, ScalarValue};use datafusion_physical_expr_common::physical_expr::format_physical_expr_list;use indexmap::{IndexMap, IndexSet};@@ -303,7 +303,7 @@ type AugmentedMapping<'a> = IndexMap<#[derive(Clone, Debug, Default)]pub struct EquivalenceGroup {/// A mapping from expressions to their equivalence class key.- map: HashMap<Arc<dyn PhysicalExpr>, usize>,+ map: IndexMap<Arc<dyn PhysicalExpr>, usize>,/// The equivalence classes in this group.classes: Vec<EquivalenceClass>,}@@ -436,7 +436,7 @@ impl EquivalenceGroup {let cls = self.classes.swap_remove(idx);// Remove its entries from the lookup table:for expr in cls.iter() {- self.map.remove(expr);+ self.map.swap_remove(expr);}// Update the lookup table for the moved class:if idx < self.classes.len() {@@ -448,7 +448,7 @@ impl EquivalenceGroup {/// Updates the entry in lookup table for the given equivalence class with/// the given index.fn update_lookup_table(- map: &mut HashMap<Arc<dyn PhysicalExpr>, usize>,+ map: &mut IndexMap<Arc<dyn PhysicalExpr>, usize>,cls: &EquivalenceClass,idx: usize,) {diff --git a/datafusion/physical-expr/src/expressions/case.rs b/datafusion/physical-expr/src/expressions/case.rsindex 582d6a141a..758317d3d2 100644--- a/datafusion/physical-expr/src/expressions/case.rs+++ b/datafusion/physical-expr/src/expressions/case.rs@@ -29,10 +29,11 @@ use arrow::datatypes::{DataType, Schema, UInt32Type, UnionMode};use arrow::error::ArrowError;use datafusion_common::cast::as_boolean_array;use datafusion_common::{- DataFusionError, HashMap, HashSet, Result, ScalarValue, assert_or_internal_err,- exec_err, internal_datafusion_err, internal_err,+ DataFusionError, Result, ScalarValue, assert_or_internal_err, exec_err,+ internal_datafusion_err, internal_err,};use datafusion_expr::ColumnarValue;+use indexmap::{IndexMap, IndexSet};use std::borrow::Cow;use std::hash::Hash;use std::{any::Any, sync::Arc};@@ -122,7 +123,7 @@ impl CaseBody {/// Derives a [ProjectedCaseBody] from this [CaseBody].fn project(&self) -> Result<ProjectedCaseBody> {// Determine the set of columns that are used in all the expressions of the case body.- let mut used_column_indices = HashSet::<usize>::new();+ let mut used_column_indices = IndexSet::<usize>::new();let mut collect_column_indices = |expr: &Arc<dyn PhysicalExpr>| {expr.apply(|expr| {if let Some(column) = expr.as_any().downcast_ref::<Column>() {@@ -149,7 +150,7 @@ impl CaseBody {.iter().enumerate().map(|(projected, original)| (*original, projected))- .collect::<HashMap<usize, usize>>();+ .collect::<IndexMap<usize, usize>>();// Construct the projected body by rewriting each expression from the original body// using the column index mapping.diff --git a/datafusion/physical-expr/src/utils/mod.rs b/datafusion/physical-expr/src/utils/mod.rsindex cd476ee3b3..2cdc326f5d 100644--- a/datafusion/physical-expr/src/utils/mod.rs+++ b/datafusion/physical-expr/src/utils/mod.rs@@ -229,7 +229,7 @@ pub fn collect_columns(expr: &Arc<dyn PhysicalExpr>) -> HashSet<Column> {let mut columns = HashSet::<Column>::new();expr.apply(|expr| {if let Some(column) = expr.as_any().downcast_ref::<Column>() {- columns.get_or_insert_owned(column);+ columns.get_or_insert_with(column, |c| c.clone());}Ok(TreeNodeRecursion::Continue)})