Skip to content

Apache DataFusion SQL Query Engine

Optimize planning / stop cloning Strings / Fields so much (2-3% faster planning time) (#18415)

Andrew Lamb477053d83120
diff --git a/datafusion/common/src/datatype.rs b/datafusion/common/src/datatype.rs
index 65f6395211..19847f8583 100644
--- a/datafusion/common/src/datatype.rs
+++ b/datafusion/common/src/datatype.rs
@@ -15,9 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-//! [`DataTypeExt`] and [`FieldExt`] extension trait for working with DataTypes to Fields
+//! [`DataTypeExt`] and [`FieldExt`] extension trait for working with Arrow [`DataType`] and [`Field`]s
use crate::arrow::datatypes::{DataType, Field, FieldRef};
+use crate::metadata::FieldMetadata;
use std::sync::Arc;
/// DataFusion extension methods for Arrow [`DataType`]
@@ -61,7 +62,54 @@ impl DataTypeExt for DataType {
}
/// DataFusion extension methods for Arrow [`Field`] and [`FieldRef`]
+///
+/// This trait is implemented for both [`Field`] and [`FieldRef`] and
+/// provides convenience methods for efficiently working with both types.
+///
+/// For [`FieldRef`], the methods will attempt to unwrap the `Arc`
+/// to avoid unnecessary cloning when possible.
pub trait FieldExt {
+ /// Ensure the field is named `new_name`, returning the given field if the
+ /// name matches, and a new field if not.
+ ///
+ /// This method avoids `clone`ing fields and names if the name is the same
+ /// as the field's existing name.
+ ///
+ /// Example:
+ /// ```
+ /// # use std::sync::Arc;
+ /// # use arrow::datatypes::{DataType, Field};
+ /// # use datafusion_common::datatype::FieldExt;
+ /// let int_field = Field::new("my_int", DataType::Int32, true);
+ /// // rename to "your_int"
+ /// let renamed_field = int_field.renamed("your_int");
+ /// assert_eq!(renamed_field.name(), "your_int");
+ /// ```
+ fn renamed(self, new_name: &str) -> Self;
+
+ /// Ensure the field has the given data type
+ ///
+ /// Note this is different than simply calling [`Field::with_data_type`] as
+ /// it avoids copying if the data type is already the same.
+ ///
+ /// Example:
+ /// ```
+ /// # use std::sync::Arc;
+ /// # use arrow::datatypes::{DataType, Field};
+ /// # use datafusion_common::datatype::FieldExt;
+ /// let int_field = Field::new("my_int", DataType::Int32, true);
+ /// // change to Float64
+ /// let retyped_field = int_field.retyped(DataType::Float64);
+ /// assert_eq!(retyped_field.data_type(), &DataType::Float64);
+ /// ```
+ fn retyped(self, new_data_type: DataType) -> Self;
+
+ /// Add field metadata to the Field
+ fn with_field_metadata(self, metadata: &FieldMetadata) -> Self;
+
+ /// Add optional field metadata,
+ fn with_field_metadata_opt(self, metadata: Option<&FieldMetadata>) -> Self;
+
/// Returns a new Field representing a List of this Field's DataType.
///
/// For example if input represents an `Int32`, the return value will
@@ -130,6 +178,32 @@ pub trait FieldExt {
}
impl FieldExt for Field {
+ fn renamed(self, new_name: &str) -> Self {
+ // check if this is a new name before allocating a new Field / copying
+ // the existing one
+ if self.name() != new_name {
+ self.with_name(new_name)
+ } else {
+ self
+ }
+ }
+
+ fn retyped(self, new_data_type: DataType) -> Self {
+ self.with_data_type(new_data_type)
+ }
+
+ fn with_field_metadata(self, metadata: &FieldMetadata) -> Self {
+ metadata.add_to_field(self)
+ }
+
+ fn with_field_metadata_opt(self, metadata: Option<&FieldMetadata>) -> Self {
+ if let Some(metadata) = metadata {
+ self.with_field_metadata(metadata)
+ } else {
+ self
+ }
+ }
+
fn into_list(self) -> Self {
DataType::List(Arc::new(self.into_list_item())).into_nullable_field()
}
@@ -149,6 +223,34 @@ impl FieldExt for Field {
}
impl FieldExt for Arc<Field> {
+ fn renamed(mut self, new_name: &str) -> Self {
+ if self.name() != new_name {
+ // avoid cloning if possible
+ Arc::make_mut(&mut self).set_name(new_name);
+ }
+ self
+ }
+
+ fn retyped(mut self, new_data_type: DataType) -> Self {
+ if self.data_type() != &new_data_type {
+ // avoid cloning if possible
+ Arc::make_mut(&mut self).set_data_type(new_data_type);
+ }
+ self
+ }
+
+ fn with_field_metadata(self, metadata: &FieldMetadata) -> Self {
+ metadata.add_to_field_ref(self)
+ }
+
+ fn with_field_metadata_opt(self, metadata: Option<&FieldMetadata>) -> Self {
+ if let Some(metadata) = metadata {
+ self.with_field_metadata(metadata)
+ } else {
+ self
+ }
+ }
+
fn into_list(self) -> Self {
DataType::List(self.into_list_item())
.into_nullable_field()
@@ -161,13 +263,11 @@ impl FieldExt for Arc<Field> {
.into()
}
- fn into_list_item(self) -> Self {
+ fn into_list_item(mut self) -> Self {
if self.name() != Field::LIST_FIELD_DEFAULT_NAME {
- Arc::unwrap_or_clone(self)
- .with_name(Field::LIST_FIELD_DEFAULT_NAME)
- .into()
- } else {
- self
+ // avoid cloning if possible
+ Arc::make_mut(&mut self).set_name(Field::LIST_FIELD_DEFAULT_NAME);
}
+ self
}
}
diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs
index fd4584bca0..668e72c759 100644
--- a/datafusion/common/src/dfschema.rs
+++ b/datafusion/common/src/dfschema.rs
@@ -37,7 +37,7 @@ use arrow::datatypes::{
/// A reference-counted reference to a [DFSchema].
pub type DFSchemaRef = Arc<DFSchema>;
-/// DFSchema wraps an Arrow schema and adds relation names.
+/// DFSchema wraps an Arrow schema and add a relation (table) name.
///
/// The schema may hold the fields across multiple tables. Some fields may be
/// qualified and some unqualified. A qualified field is a field that has a
@@ -47,8 +47,14 @@ pub type DFSchemaRef = Arc<DFSchema>;
/// have a distinct name from any qualified field names. This allows finding a
/// qualified field by name to be possible, so long as there aren't multiple
/// qualified fields with the same name.
+///]
+/// # See Also
+/// * [DFSchemaRef], an alias to `Arc<DFSchema>`
+/// * [DataTypeExt], common methods for working with Arrow [DataType]s
+/// * [FieldExt], extension methods for working with Arrow [Field]s
///
-/// There is an alias to `Arc<DFSchema>` named [DFSchemaRef].
+/// [DataTypeExt]: crate::datatype::DataTypeExt
+/// [FieldExt]: crate::datatype::FieldExt
///
/// # Creating qualified schemas
///
@@ -346,20 +352,22 @@ impl DFSchema {
self.field_qualifiers.extend(qualifiers);
}
- /// Get a list of fields
+ /// Get a list of fields for this schema
pub fn fields(&self) -> &Fields {
&self.inner.fields
}
- /// Returns an immutable reference of a specific `Field` instance selected using an
- /// offset within the internal `fields` vector
- pub fn field(&self, i: usize) -> &Field {
+ /// Returns a reference to [`FieldRef`] for a column at specific index
+ /// within the schema.
+ ///
+ /// See also [Self::qualified_field] to get both qualifier and field
+ pub fn field(&self, i: usize) -> &FieldRef {
&self.inner.fields[i]
}
- /// Returns an immutable reference of a specific `Field` instance selected using an
- /// offset within the internal `fields` vector and its qualifier
- pub fn qualified_field(&self, i: usize) -> (Option<&TableReference>, &Field) {
+ /// Returns the qualifier (if any) and [`FieldRef`] for a column at specific
+ /// index within the schema.
+ pub fn qualified_field(&self, i: usize) -> (Option<&TableReference>, &FieldRef) {
(self.field_qualifiers[i].as_ref(), self.field(i))
}
@@ -410,12 +418,12 @@ impl DFSchema {
.is_some()
}
- /// Find the field with the given name
+ /// Find the [`FieldRef`] with the given name and optional qualifier
pub fn field_with_name(
&self,
qualifier: Option<&TableReference>,
name: &str,
- ) -> Result<&Field> {
+ ) -> Result<&FieldRef> {
if let Some(qualifier) = qualifier {
self.field_with_qualified_name(qualifier, name)
} else {
@@ -428,7 +436,7 @@ impl DFSchema {
&self,
qualifier: Option<&TableReference>,
name: &str,
- ) -> Result<(Option<&TableReference>, &Field)> {
+ ) -> Result<(Option<&TableReference>, &FieldRef)> {
if let Some(qualifier) = qualifier {
let idx = self
.index_of_column_by_name(Some(qualifier), name)
@@ -440,10 +448,10 @@ impl DFSchema {
}
/// Find all fields having the given qualifier
- pub fn fields_with_qualified(&self, qualifier: &TableReference) -> Vec<&Field> {
+ pub fn fields_with_qualified(&self, qualifier: &TableReference) -> Vec<&FieldRef> {
self.iter()
.filter(|(q, _)| q.map(|q| q.eq(qualifier)).unwrap_or(false))
- .map(|(_, f)| f.as_ref())
+ .map(|(_, f)| f)
.collect()
}
@@ -459,11 +467,10 @@ impl DFSchema {
}
/// Find all fields that match the given name
- pub fn fields_with_unqualified_name(&self, name: &str) -> Vec<&Field> {
+ pub fn fields_with_unqualified_name(&self, name: &str) -> Vec<&FieldRef> {
self.fields()
.iter()
.filter(|field| field.name() == name)
- .map(|f| f.as_ref())
.collect()
}
@@ -471,10 +478,9 @@ impl DFSchema {
pub fn qualified_fields_with_unqualified_name(
&self,
name: &str,
- ) -> Vec<(Option<&TableReference>, &Field)> {
+ ) -> Vec<(Option<&TableReference>, &FieldRef)> {
self.iter()
.filter(|(_, field)| field.name() == name)
- .map(|(qualifier, field)| (qualifier, field.as_ref()))
.collect()
}
@@ -499,7 +505,7 @@ impl DFSchema {
pub fn qualified_field_with_unqualified_name(
&self,
name: &str,
- ) -> Result<(Option<&TableReference>, &Field)> {
+ ) -> Result<(Option<&TableReference>, &FieldRef)> {
let matches = self.qualified_fields_with_unqualified_name(name);
match matches.len() {
0 => Err(unqualified_field_not_found(name, self)),
@@ -528,7 +534,7 @@ impl DFSchema {
}
/// Find the field with the given name
- pub fn field_with_unqualified_name(&self, name: &str) -> Result<&Field> {
+ pub fn field_with_unqualified_name(&self, name: &str) -> Result<&FieldRef> {
self.qualified_field_with_unqualified_name(name)
.map(|(_, field)| field)
}
@@ -538,7 +544,7 @@ impl DFSchema {
&self,
qualifier: &TableReference,
name: &str,
- ) -> Result<&Field> {
+ ) -> Result<&FieldRef> {
let idx = self
.index_of_column_by_name(Some(qualifier), name)
.ok_or_else(|| field_not_found(Some(qualifier.clone()), name, self))?;
@@ -550,7 +556,7 @@ impl DFSchema {
pub fn qualified_field_from_column(
&self,
column: &Column,
- ) -> Result<(Option<&TableReference>, &Field)> {
+ ) -> Result<(Option<&TableReference>, &FieldRef)> {
self.qualified_field_with_name(column.relation.as_ref(), &column.name)
}
@@ -1220,7 +1226,7 @@ pub trait ExprSchema: std::fmt::Debug {
}
// Return the column's field
- fn field_from_column(&self, col: &Column) -> Result<&Field>;
+ fn field_from_column(&self, col: &Column) -> Result<&FieldRef>;
}
// Implement `ExprSchema` for `Arc<DFSchema>`
@@ -1241,13 +1247,13 @@ impl<P: AsRef<DFSchema> + std::fmt::Debug> ExprSchema for P {
self.as_ref().data_type_and_nullable(col)
}
- fn field_from_column(&self, col: &Column) -> Result<&Field> {
+ fn field_from_column(&self, col: &Column) -> Result<&FieldRef> {
self.as_ref().field_from_column(col)
}
}
impl ExprSchema for DFSchema {
- fn field_from_column(&self, col: &Column) -> Result<&Field> {
+ fn field_from_column(&self, col: &Column) -> Result<&FieldRef> {
match &col.relation {
Some(r) => self.field_with_qualified_name(r, &col.name),
None => self.field_with_unqualified_name(&col.name),
diff --git a/datafusion/common/src/metadata.rs b/datafusion/common/src/metadata.rs
index df5e70ff5b..eb687bde07 100644
--- a/datafusion/common/src/metadata.rs
+++ b/datafusion/common/src/metadata.rs
@@ -17,7 +17,7 @@
use std::{collections::BTreeMap, sync::Arc};
-use arrow::datatypes::{DataType, Field};
+use arrow::datatypes::{DataType, Field, FieldRef};
use hashbrown::HashMap;
use crate::{DataFusionError, ScalarValue, error::_plan_err};
@@ -320,6 +320,16 @@ impl FieldMetadata {
field.with_metadata(self.to_hashmap())
}
+
+ /// Updates the metadata on the FieldRef with this metadata, if it is not empty.
+ pub fn add_to_field_ref(&self, mut field_ref: FieldRef) -> FieldRef {
+ if self.inner.is_empty() {
+ return field_ref;
+ }
+
+ Arc::make_mut(&mut field_ref).set_metadata(self.to_hashmap());
+ field_ref
+ }
}
impl From<&Field> for FieldMetadata {
diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs
index 8e52ca1bf1..66dbd89f86 100644
--- a/datafusion/core/src/dataframe/mod.rs
+++ b/datafusion/core/src/dataframe/mod.rs
@@ -49,6 +49,7 @@ use std::sync::Arc;
use arrow::array::{Array, ArrayRef, Int64Array, StringArray};
use arrow::compute::{cast, concat};
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use arrow_schema::FieldRef;
use datafusion_common::config::{CsvOptions, JsonOptions};
use datafusion_common::{
exec_err, internal_datafusion_err, not_impl_err, plan_datafusion_err, plan_err,
@@ -2241,7 +2242,7 @@ impl DataFrame {
.schema()
.iter()
.map(|(qualifier, field)| {
- if qualifier.eq(&qualifier_rename) && field.as_ref() == field_rename {
+ if qualifier.eq(&qualifier_rename) && field == field_rename {
(
col(Column::from((qualifier, field)))
.alias_qualified(qualifier.cloned(), new_name),
@@ -2413,7 +2414,7 @@ impl DataFrame {
.schema()
.fields()
.iter()
- .map(|f| f.as_ref().clone())
+ .map(Arc::clone)
.collect()
} else {
self.find_columns(&columns)?
@@ -2450,7 +2451,7 @@ impl DataFrame {
}
// Helper to find columns from names
- fn find_columns(&self, names: &[String]) -> Result<Vec<Field>> {
+ fn find_columns(&self, names: &[String]) -> Result<Vec<FieldRef>> {
let schema = self.logical_plan().schema();
names
.iter()
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
index aea530d67f..9e4c78a3d0 100644
--- a/datafusion/core/src/physical_planner.rs
+++ b/datafusion/core/src/physical_planner.rs
@@ -1213,7 +1213,7 @@ impl DefaultPhysicalPlanner {
let filter_df_fields = filter_df_fields
.into_iter()
.map(|(qualifier, field)| {
- (qualifier.cloned(), Arc::new(field.clone()))
+ (qualifier.cloned(), Arc::clone(field))
})
.collect();
diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rs
index daf0c4f0df..993b806fde 100644
--- a/datafusion/expr/src/expr_schema.rs
+++ b/datafusion/expr/src/expr_schema.rs
@@ -28,6 +28,7 @@ use crate::udf::ReturnFieldArgs;
use crate::{utils, LogicalPlan, Projection, Subquery, WindowFunctionDefinition};
use arrow::compute::can_cast_types;
use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::datatype::FieldExt;
use datafusion_common::metadata::FieldMetadata;
use datafusion_common::{
not_impl_err, plan_datafusion_err, plan_err, Column, DataFusionError, ExprSchema,
@@ -485,7 +486,7 @@ impl ExprSchemable for Expr {
schema: &dyn ExprSchema,
) -> Result<(Option<TableReference>, Arc<Field>)> {
let (relation, schema_name) = self.qualified_name();
- #[allow(deprecated)]
+ #[expect(deprecated)]
let field = match self {
Expr::Alias(Alias {
expr,
@@ -493,30 +494,28 @@ impl ExprSchemable for Expr {
metadata,
..
}) => {
- let field = expr.to_field(schema).map(|(_, f)| f.as_ref().clone())?;
-
let mut combined_metadata = expr.metadata(schema)?;
if let Some(metadata) = metadata {
combined_metadata.extend(metadata.clone());
}
- Ok(Arc::new(combined_metadata.add_to_field(field)))
+ Ok(expr
+ .to_field(schema)
+ .map(|(_, f)| f)?
+ .with_field_metadata(&combined_metadata))
}
Expr::Negative(expr) => expr.to_field(schema).map(|(_, f)| f),
- Expr::Column(c) => schema.field_from_column(c).map(|f| Arc::new(f.clone())),
+ Expr::Column(c) => schema.field_from_column(c).map(Arc::clone),
Expr::OuterReferenceColumn(field, _) => {
- Ok(Arc::new(field.as_ref().clone().with_name(&schema_name)))
+ Ok(Arc::clone(field).renamed(&schema_name))
}
Expr::ScalarVariable(ty, _) => {
Ok(Arc::new(Field::new(&schema_name, ty.clone(), true)))
}
- Expr::Literal(l, metadata) => {
- let mut field = Field::new(&schema_name, l.data_type(), l.is_null());
- if let Some(metadata) = metadata {
- field = metadata.add_to_field(field);
- }
- Ok(Arc::new(field))
- }
+ Expr::Literal(l, metadata) => Ok(Arc::new(
+ Field::new(&schema_name, l.data_type(), l.is_null())
+ .with_field_metadata_opt(metadata.as_ref()),
+ )),
Expr::IsNull(_)
| Expr::IsNotNull(_)
| Expr::IsTrue(_)
@@ -623,8 +622,7 @@ impl ExprSchemable for Expr {
let new_fields = fields
.into_iter()
.zip(new_data_types)
- .map(|(f, d)| f.as_ref().clone().with_data_type(d))
- .map(Arc::new)
+ .map(|(f, d)| f.retyped(d))
.collect::<Vec<FieldRef>>();
let arguments = args
@@ -644,12 +642,11 @@ impl ExprSchemable for Expr {
// _ => Ok((self.get_type(schema)?, self.nullable(schema)?)),
Expr::Cast(Cast { expr, data_type }) => expr
.to_field(schema)
- .map(|(_, f)| f.as_ref().clone().with_data_type(data_type.clone()))
- .map(Arc::new),
+ .map(|(_, f)| f.retyped(data_type.clone())),
Expr::Placeholder(Placeholder {
id: _,
field: Some(field),
- }) => Ok(field.as_ref().clone().with_name(&schema_name).into()),
+ }) => Ok(Arc::clone(field).renamed(&schema_name)),
Expr::Like(_)
| Expr::SimilarTo(_)
| Expr::Not(_)
@@ -670,7 +667,8 @@ impl ExprSchemable for Expr {
Ok((
relation,
- Arc::new(field.as_ref().clone().with_name(schema_name)),
+ // todo avoid this rename / use the name above
+ field.renamed(&schema_name),
))
}
@@ -1172,25 +1170,25 @@ mod tests {
#[derive(Debug)]
struct MockExprSchema {
- field: Field,
+ field: FieldRef,
error_on_nullable: bool,
}
impl MockExprSchema {
fn new() -> Self {
Self {
- field: Field::new("mock_field", DataType::Null, false),
+ field: Arc::new(Field::new("mock_field", DataType::Null, false)),
error_on_nullable: false,
}
}
fn with_nullable(mut self, nullable: bool) -> Self {
- self.field = self.field.with_nullable(nullable);
+ Arc::make_mut(&mut self.field).set_nullable(nullable);
self
}
fn with_data_type(mut self, data_type: DataType) -> Self {
- self.field = self.field.with_data_type(data_type);
+ Arc::make_mut(&mut self.field).set_data_type(data_type);
self
}
@@ -1200,7 +1198,8 @@ mod tests {
}
fn with_metadata(mut self, metadata: FieldMetadata) -> Self {
- self.field = metadata.add_to_field(self.field);
+ self.field =
+ Arc::new(metadata.add_to_field(Arc::unwrap_or_clone(self.field)));
self
}
}
@@ -1211,7 +1210,7 @@ mod tests {
Ok(self.field.is_nullable())
}
- fn field_from_column(&self, _col: &Column) -> Result<&Field> {
+ fn field_from_column(&self, _col: &Column) -> Result<&FieldRef> {
Ok(&self.field)
}
}
diff --git a/datafusion/sql/src/expr/identifier.rs b/datafusion/sql/src/expr/identifier.rs
index b5db580217..b329cc5d1f 100644
--- a/datafusion/sql/src/expr/identifier.rs
+++ b/datafusion/sql/src/expr/identifier.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::datatypes::Field;
+use arrow::datatypes::FieldRef;
use datafusion_common::{
assert_or_internal_err, exec_datafusion_err, internal_err, not_impl_err,
plan_datafusion_err, plan_err, Column, DFSchema, Result, Span, TableReference,
@@ -76,7 +76,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
{
// Found an exact match on a qualified name in the outer plan schema, so this is an outer reference column
return Ok(Expr::OuterReferenceColumn(
- Arc::new(field.clone()),
+ Arc::clone(field),
Column::from((qualifier, field)),
));
}
@@ -179,7 +179,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
Some((field, qualifier, _nested_names)) => {
// Found an exact match on a qualified name in the outer plan schema, so this is an outer reference column
Ok(Expr::OuterReferenceColumn(
- Arc::new(field.clone()),
+ Arc::clone(field),
Column::from((qualifier, field)),
))
}
@@ -291,7 +291,7 @@ fn search_dfschema<'ids, 'schema>(
ids: &'ids [String],
schema: &'schema DFSchema,
) -> Option<(
- &'schema Field,
+ &'schema FieldRef,
Option<&'schema TableReference>,
&'ids [String],
)> {
diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs
index e89ddaa349..a30c6c31fa 100644
--- a/datafusion/sql/src/statement.rs
+++ b/datafusion/sql/src/statement.rs
@@ -2166,7 +2166,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
} else {
value_indices[column_index] = Some(i);
}
- Ok(table_schema.field(column_index).clone())
+ Ok(Arc::clone(table_schema.field(column_index)))
})
.collect::<Result<Vec<_>>>()?;
(Fields::from(fields), value_indices)
diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
index 48e93c04bb..5f2a4bca4b 100644
--- a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
@@ -242,9 +242,7 @@ pub fn apply_masking(
let fields = column_indices
.iter()
.map(|i| schema.qualified_field(*i))
- .map(|(qualifier, field)| {
- (qualifier.cloned(), Arc::new(field.clone()))
- })
+ .map(|(qualifier, field)| (qualifier.cloned(), Arc::clone(field)))
.collect();
Ok(DFSchema::new_with_metadata(
@@ -288,7 +286,7 @@ fn apply_projection(
let fields = column_indices
.iter()
.map(|i| df_schema.qualified_field(*i))
- .map(|(qualifier, field)| (qualifier.cloned(), Arc::new(field.clone())))
+ .map(|(qualifier, field)| (qualifier.cloned(), Arc::clone(field)))
.collect();
scan.projected_schema = DFSchemaRef::new(DFSchema::new_with_metadata(
diff --git a/datafusion/substrait/src/logical_plan/consumer/utils.rs b/datafusion/substrait/src/logical_plan/consumer/utils.rs
index f7eedcb7a2..993a92212b 100644
--- a/datafusion/substrait/src/logical_plan/consumer/utils.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/utils.rs
@@ -577,12 +577,12 @@ pub(crate) mod tests {
assert_eq!(renamed_schema.fields().len(), 5);
assert_eq!(
- *renamed_schema.field(0),
- Field::new("a", DataType::Int32, false)
+ renamed_schema.field(0),
+ &Arc::new(Field::new("a", DataType::Int32, false))
);
assert_eq!(
- *renamed_schema.field(1),
- Field::new_struct(
+ renamed_schema.field(1),
+ &Arc::new(Field::new_struct(
"b",
vec![
Field::new("c", DataType::Int32, false),
@@ -593,11 +593,11 @@ pub(crate) mod tests {
)
],
false,
- )
+ ))
);
assert_eq!(
- *renamed_schema.field(2),
- Field::new_list(
+ renamed_schema.field(2),
+ &Arc::new(Field::new_list(
"f",
Arc::new(Field::new_struct(
"item",
@@ -605,11 +605,11 @@ pub(crate) mod tests {
false,
)),
false,
- )
+ ))
);
assert_eq!(
- *renamed_schema.field(3),
- Field::new_large_list(
+ renamed_schema.field(3),
+ &Arc::new(Field::new_large_list(
"h",
Arc::new(Field::new_struct(
"item",
@@ -617,11 +617,11 @@ pub(crate) mod tests {
false,
)),
false,
- )
+ ))
);
assert_eq!(
- *renamed_schema.field(4),
- Field::new_map(
+ renamed_schema.field(4),
+ &Arc::new(Field::new_map(
"j",
"entries",
Arc::new(Field::new_struct(
@@ -636,7 +636,7 @@ pub(crate) mod tests {
)),
false,
false,
- )
+ ))
);
Ok(())
}
diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.md
index 25c209c5eb..d40fee0938 100644
--- a/docs/source/library-user-guide/upgrading.md
+++ b/docs/source/library-user-guide/upgrading.md
@@ -25,6 +25,21 @@
You can see the current [status of the `52.0.0`release here](https://github.com/apache/datafusion/issues/18566)
+### Changes to DFSchema API
+
+To permit more efficient planning, several methods on `DFSchema` have been
+changed to return references to the underlying [`&FieldRef`] rather than
+[`&Field`]. This allows planners to more cheaply copy the references via
+`Arc::clone` rather than cloning the entire `Field` structure.
+
+You may need to change code to use `Arc::clone` instead of `.as_ref().clone()`
+directly on the `Field`. For example:
+
+```diff
+- let field = df_schema.field("my_column").as_ref().clone();
++ let field = Arc::clone(df_schema.field("my_column"));
+```
+
### Removal of `pyarrow` feature
The `pyarrow` feature flag has been removed. This feature has been migrated to