Apache DataFusion SQL Query Engine
Optimize planning / stop cloning Strings / Fields so much (2-3% faster planning time) (#18415)
Andrew Lamb477053d83120
diff --git a/datafusion/common/src/datatype.rs b/datafusion/common/src/datatype.rsindex 65f6395211..19847f8583 100644--- a/datafusion/common/src/datatype.rs+++ b/datafusion/common/src/datatype.rs@@ -15,9 +15,10 @@// specific language governing permissions and limitations// under the License.-//! [`DataTypeExt`] and [`FieldExt`] extension trait for working with DataTypes to Fields+//! [`DataTypeExt`] and [`FieldExt`] extension trait for working with Arrow [`DataType`] and [`Field`]suse crate::arrow::datatypes::{DataType, Field, FieldRef};+use crate::metadata::FieldMetadata;use std::sync::Arc;/// DataFusion extension methods for Arrow [`DataType`]@@ -61,7 +62,54 @@ impl DataTypeExt for DataType {}/// DataFusion extension methods for Arrow [`Field`] and [`FieldRef`]+///+/// This trait is implemented for both [`Field`] and [`FieldRef`] and+/// provides convenience methods for efficiently working with both types.+///+/// For [`FieldRef`], the methods will attempt to unwrap the `Arc`+/// to avoid unnecessary cloning when possible.pub trait FieldExt {+ /// Ensure the field is named `new_name`, returning the given field if the+ /// name matches, and a new field if not.+ ///+ /// This method avoids `clone`ing fields and names if the name is the same+ /// as the field's existing name.+ ///+ /// Example:+ /// ```+ /// # use std::sync::Arc;+ /// # use arrow::datatypes::{DataType, Field};+ /// # use datafusion_common::datatype::FieldExt;+ /// let int_field = Field::new("my_int", DataType::Int32, true);+ /// // rename to "your_int"+ /// let renamed_field = int_field.renamed("your_int");+ /// assert_eq!(renamed_field.name(), "your_int");+ /// ```+ fn renamed(self, new_name: &str) -> Self;++ /// Ensure the field has the given data type+ ///+ /// Note this is different than simply calling [`Field::with_data_type`] as+ /// it avoids copying if the data type is already the same.+ ///+ /// Example:+ /// ```+ /// # use std::sync::Arc;+ /// # use arrow::datatypes::{DataType, Field};+ /// # use datafusion_common::datatype::FieldExt;+ /// let int_field = Field::new("my_int", DataType::Int32, true);+ /// // change to Float64+ /// let retyped_field = int_field.retyped(DataType::Float64);+ /// assert_eq!(retyped_field.data_type(), &DataType::Float64);+ /// ```+ fn retyped(self, new_data_type: DataType) -> Self;++ /// Add field metadata to the Field+ fn with_field_metadata(self, metadata: &FieldMetadata) -> Self;++ /// Add optional field metadata,+ fn with_field_metadata_opt(self, metadata: Option<&FieldMetadata>) -> Self;+/// Returns a new Field representing a List of this Field's DataType.////// For example if input represents an `Int32`, the return value will@@ -130,6 +178,32 @@ pub trait FieldExt {}impl FieldExt for Field {+ fn renamed(self, new_name: &str) -> Self {+ // check if this is a new name before allocating a new Field / copying+ // the existing one+ if self.name() != new_name {+ self.with_name(new_name)+ } else {+ self+ }+ }++ fn retyped(self, new_data_type: DataType) -> Self {+ self.with_data_type(new_data_type)+ }++ fn with_field_metadata(self, metadata: &FieldMetadata) -> Self {+ metadata.add_to_field(self)+ }++ fn with_field_metadata_opt(self, metadata: Option<&FieldMetadata>) -> Self {+ if let Some(metadata) = metadata {+ self.with_field_metadata(metadata)+ } else {+ self+ }+ }+fn into_list(self) -> Self {DataType::List(Arc::new(self.into_list_item())).into_nullable_field()}@@ -149,6 +223,34 @@ impl FieldExt for Field {}impl FieldExt for Arc<Field> {+ fn renamed(mut self, new_name: &str) -> Self {+ if self.name() != new_name {+ // avoid cloning if possible+ Arc::make_mut(&mut self).set_name(new_name);+ }+ self+ }++ fn retyped(mut self, new_data_type: DataType) -> Self {+ if self.data_type() != &new_data_type {+ // avoid cloning if possible+ Arc::make_mut(&mut self).set_data_type(new_data_type);+ }+ self+ }++ fn with_field_metadata(self, metadata: &FieldMetadata) -> Self {+ metadata.add_to_field_ref(self)+ }++ fn with_field_metadata_opt(self, metadata: Option<&FieldMetadata>) -> Self {+ if let Some(metadata) = metadata {+ self.with_field_metadata(metadata)+ } else {+ self+ }+ }+fn into_list(self) -> Self {DataType::List(self.into_list_item()).into_nullable_field()@@ -161,13 +263,11 @@ impl FieldExt for Arc<Field> {.into()}- fn into_list_item(self) -> Self {+ fn into_list_item(mut self) -> Self {if self.name() != Field::LIST_FIELD_DEFAULT_NAME {- Arc::unwrap_or_clone(self)- .with_name(Field::LIST_FIELD_DEFAULT_NAME)- .into()- } else {- self+ // avoid cloning if possible+ Arc::make_mut(&mut self).set_name(Field::LIST_FIELD_DEFAULT_NAME);}+ self}}diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rsindex fd4584bca0..668e72c759 100644--- a/datafusion/common/src/dfschema.rs+++ b/datafusion/common/src/dfschema.rs@@ -37,7 +37,7 @@ use arrow::datatypes::{/// A reference-counted reference to a [DFSchema].pub type DFSchemaRef = Arc<DFSchema>;-/// DFSchema wraps an Arrow schema and adds relation names.+/// DFSchema wraps an Arrow schema and add a relation (table) name.////// The schema may hold the fields across multiple tables. Some fields may be/// qualified and some unqualified. A qualified field is a field that has a@@ -47,8 +47,14 @@ pub type DFSchemaRef = Arc<DFSchema>;/// have a distinct name from any qualified field names. This allows finding a/// qualified field by name to be possible, so long as there aren't multiple/// qualified fields with the same name.+///]+/// # See Also+/// * [DFSchemaRef], an alias to `Arc<DFSchema>`+/// * [DataTypeExt], common methods for working with Arrow [DataType]s+/// * [FieldExt], extension methods for working with Arrow [Field]s///-/// There is an alias to `Arc<DFSchema>` named [DFSchemaRef].+/// [DataTypeExt]: crate::datatype::DataTypeExt+/// [FieldExt]: crate::datatype::FieldExt////// # Creating qualified schemas///@@ -346,20 +352,22 @@ impl DFSchema {self.field_qualifiers.extend(qualifiers);}- /// Get a list of fields+ /// Get a list of fields for this schemapub fn fields(&self) -> &Fields {&self.inner.fields}- /// Returns an immutable reference of a specific `Field` instance selected using an- /// offset within the internal `fields` vector- pub fn field(&self, i: usize) -> &Field {+ /// Returns a reference to [`FieldRef`] for a column at specific index+ /// within the schema.+ ///+ /// See also [Self::qualified_field] to get both qualifier and field+ pub fn field(&self, i: usize) -> &FieldRef {&self.inner.fields[i]}- /// Returns an immutable reference of a specific `Field` instance selected using an- /// offset within the internal `fields` vector and its qualifier- pub fn qualified_field(&self, i: usize) -> (Option<&TableReference>, &Field) {+ /// Returns the qualifier (if any) and [`FieldRef`] for a column at specific+ /// index within the schema.+ pub fn qualified_field(&self, i: usize) -> (Option<&TableReference>, &FieldRef) {(self.field_qualifiers[i].as_ref(), self.field(i))}@@ -410,12 +418,12 @@ impl DFSchema {.is_some()}- /// Find the field with the given name+ /// Find the [`FieldRef`] with the given name and optional qualifierpub fn field_with_name(&self,qualifier: Option<&TableReference>,name: &str,- ) -> Result<&Field> {+ ) -> Result<&FieldRef> {if let Some(qualifier) = qualifier {self.field_with_qualified_name(qualifier, name)} else {@@ -428,7 +436,7 @@ impl DFSchema {&self,qualifier: Option<&TableReference>,name: &str,- ) -> Result<(Option<&TableReference>, &Field)> {+ ) -> Result<(Option<&TableReference>, &FieldRef)> {if let Some(qualifier) = qualifier {let idx = self.index_of_column_by_name(Some(qualifier), name)@@ -440,10 +448,10 @@ impl DFSchema {}/// Find all fields having the given qualifier- pub fn fields_with_qualified(&self, qualifier: &TableReference) -> Vec<&Field> {+ pub fn fields_with_qualified(&self, qualifier: &TableReference) -> Vec<&FieldRef> {self.iter().filter(|(q, _)| q.map(|q| q.eq(qualifier)).unwrap_or(false))- .map(|(_, f)| f.as_ref())+ .map(|(_, f)| f).collect()}@@ -459,11 +467,10 @@ impl DFSchema {}/// Find all fields that match the given name- pub fn fields_with_unqualified_name(&self, name: &str) -> Vec<&Field> {+ pub fn fields_with_unqualified_name(&self, name: &str) -> Vec<&FieldRef> {self.fields().iter().filter(|field| field.name() == name)- .map(|f| f.as_ref()).collect()}@@ -471,10 +478,9 @@ impl DFSchema {pub fn qualified_fields_with_unqualified_name(&self,name: &str,- ) -> Vec<(Option<&TableReference>, &Field)> {+ ) -> Vec<(Option<&TableReference>, &FieldRef)> {self.iter().filter(|(_, field)| field.name() == name)- .map(|(qualifier, field)| (qualifier, field.as_ref())).collect()}@@ -499,7 +505,7 @@ impl DFSchema {pub fn qualified_field_with_unqualified_name(&self,name: &str,- ) -> Result<(Option<&TableReference>, &Field)> {+ ) -> Result<(Option<&TableReference>, &FieldRef)> {let matches = self.qualified_fields_with_unqualified_name(name);match matches.len() {0 => Err(unqualified_field_not_found(name, self)),@@ -528,7 +534,7 @@ impl DFSchema {}/// Find the field with the given name- pub fn field_with_unqualified_name(&self, name: &str) -> Result<&Field> {+ pub fn field_with_unqualified_name(&self, name: &str) -> Result<&FieldRef> {self.qualified_field_with_unqualified_name(name).map(|(_, field)| field)}@@ -538,7 +544,7 @@ impl DFSchema {&self,qualifier: &TableReference,name: &str,- ) -> Result<&Field> {+ ) -> Result<&FieldRef> {let idx = self.index_of_column_by_name(Some(qualifier), name).ok_or_else(|| field_not_found(Some(qualifier.clone()), name, self))?;@@ -550,7 +556,7 @@ impl DFSchema {pub fn qualified_field_from_column(&self,column: &Column,- ) -> Result<(Option<&TableReference>, &Field)> {+ ) -> Result<(Option<&TableReference>, &FieldRef)> {self.qualified_field_with_name(column.relation.as_ref(), &column.name)}@@ -1220,7 +1226,7 @@ pub trait ExprSchema: std::fmt::Debug {}// Return the column's field- fn field_from_column(&self, col: &Column) -> Result<&Field>;+ fn field_from_column(&self, col: &Column) -> Result<&FieldRef>;}// Implement `ExprSchema` for `Arc<DFSchema>`@@ -1241,13 +1247,13 @@ impl<P: AsRef<DFSchema> + std::fmt::Debug> ExprSchema for P {self.as_ref().data_type_and_nullable(col)}- fn field_from_column(&self, col: &Column) -> Result<&Field> {+ fn field_from_column(&self, col: &Column) -> Result<&FieldRef> {self.as_ref().field_from_column(col)}}impl ExprSchema for DFSchema {- fn field_from_column(&self, col: &Column) -> Result<&Field> {+ fn field_from_column(&self, col: &Column) -> Result<&FieldRef> {match &col.relation {Some(r) => self.field_with_qualified_name(r, &col.name),None => self.field_with_unqualified_name(&col.name),diff --git a/datafusion/common/src/metadata.rs b/datafusion/common/src/metadata.rsindex df5e70ff5b..eb687bde07 100644--- a/datafusion/common/src/metadata.rs+++ b/datafusion/common/src/metadata.rs@@ -17,7 +17,7 @@use std::{collections::BTreeMap, sync::Arc};-use arrow::datatypes::{DataType, Field};+use arrow::datatypes::{DataType, Field, FieldRef};use hashbrown::HashMap;use crate::{DataFusionError, ScalarValue, error::_plan_err};@@ -320,6 +320,16 @@ impl FieldMetadata {field.with_metadata(self.to_hashmap())}++ /// Updates the metadata on the FieldRef with this metadata, if it is not empty.+ pub fn add_to_field_ref(&self, mut field_ref: FieldRef) -> FieldRef {+ if self.inner.is_empty() {+ return field_ref;+ }++ Arc::make_mut(&mut field_ref).set_metadata(self.to_hashmap());+ field_ref+ }}impl From<&Field> for FieldMetadata {diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex 8e52ca1bf1..66dbd89f86 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -49,6 +49,7 @@ use std::sync::Arc;use arrow::array::{Array, ArrayRef, Int64Array, StringArray};use arrow::compute::{cast, concat};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use arrow_schema::FieldRef;use datafusion_common::config::{CsvOptions, JsonOptions};use datafusion_common::{exec_err, internal_datafusion_err, not_impl_err, plan_datafusion_err, plan_err,@@ -2241,7 +2242,7 @@ impl DataFrame {.schema().iter().map(|(qualifier, field)| {- if qualifier.eq(&qualifier_rename) && field.as_ref() == field_rename {+ if qualifier.eq(&qualifier_rename) && field == field_rename {(col(Column::from((qualifier, field))).alias_qualified(qualifier.cloned(), new_name),@@ -2413,7 +2414,7 @@ impl DataFrame {.schema().fields().iter()- .map(|f| f.as_ref().clone())+ .map(Arc::clone).collect()} else {self.find_columns(&columns)?@@ -2450,7 +2451,7 @@ impl DataFrame {}// Helper to find columns from names- fn find_columns(&self, names: &[String]) -> Result<Vec<Field>> {+ fn find_columns(&self, names: &[String]) -> Result<Vec<FieldRef>> {let schema = self.logical_plan().schema();names.iter()diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex aea530d67f..9e4c78a3d0 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -1213,7 +1213,7 @@ impl DefaultPhysicalPlanner {let filter_df_fields = filter_df_fields.into_iter().map(|(qualifier, field)| {- (qualifier.cloned(), Arc::new(field.clone()))+ (qualifier.cloned(), Arc::clone(field))}).collect();diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rsindex daf0c4f0df..993b806fde 100644--- a/datafusion/expr/src/expr_schema.rs+++ b/datafusion/expr/src/expr_schema.rs@@ -28,6 +28,7 @@ use crate::udf::ReturnFieldArgs;use crate::{utils, LogicalPlan, Projection, Subquery, WindowFunctionDefinition};use arrow::compute::can_cast_types;use arrow::datatypes::{DataType, Field, FieldRef};+use datafusion_common::datatype::FieldExt;use datafusion_common::metadata::FieldMetadata;use datafusion_common::{not_impl_err, plan_datafusion_err, plan_err, Column, DataFusionError, ExprSchema,@@ -485,7 +486,7 @@ impl ExprSchemable for Expr {schema: &dyn ExprSchema,) -> Result<(Option<TableReference>, Arc<Field>)> {let (relation, schema_name) = self.qualified_name();- #[allow(deprecated)]+ #[expect(deprecated)]let field = match self {Expr::Alias(Alias {expr,@@ -493,30 +494,28 @@ impl ExprSchemable for Expr {metadata,..}) => {- let field = expr.to_field(schema).map(|(_, f)| f.as_ref().clone())?;-let mut combined_metadata = expr.metadata(schema)?;if let Some(metadata) = metadata {combined_metadata.extend(metadata.clone());}- Ok(Arc::new(combined_metadata.add_to_field(field)))+ Ok(expr+ .to_field(schema)+ .map(|(_, f)| f)?+ .with_field_metadata(&combined_metadata))}Expr::Negative(expr) => expr.to_field(schema).map(|(_, f)| f),- Expr::Column(c) => schema.field_from_column(c).map(|f| Arc::new(f.clone())),+ Expr::Column(c) => schema.field_from_column(c).map(Arc::clone),Expr::OuterReferenceColumn(field, _) => {- Ok(Arc::new(field.as_ref().clone().with_name(&schema_name)))+ Ok(Arc::clone(field).renamed(&schema_name))}Expr::ScalarVariable(ty, _) => {Ok(Arc::new(Field::new(&schema_name, ty.clone(), true)))}- Expr::Literal(l, metadata) => {- let mut field = Field::new(&schema_name, l.data_type(), l.is_null());- if let Some(metadata) = metadata {- field = metadata.add_to_field(field);- }- Ok(Arc::new(field))- }+ Expr::Literal(l, metadata) => Ok(Arc::new(+ Field::new(&schema_name, l.data_type(), l.is_null())+ .with_field_metadata_opt(metadata.as_ref()),+ )),Expr::IsNull(_)| Expr::IsNotNull(_)| Expr::IsTrue(_)@@ -623,8 +622,7 @@ impl ExprSchemable for Expr {let new_fields = fields.into_iter().zip(new_data_types)- .map(|(f, d)| f.as_ref().clone().with_data_type(d))- .map(Arc::new)+ .map(|(f, d)| f.retyped(d)).collect::<Vec<FieldRef>>();let arguments = args@@ -644,12 +642,11 @@ impl ExprSchemable for Expr {// _ => Ok((self.get_type(schema)?, self.nullable(schema)?)),Expr::Cast(Cast { expr, data_type }) => expr.to_field(schema)- .map(|(_, f)| f.as_ref().clone().with_data_type(data_type.clone()))- .map(Arc::new),+ .map(|(_, f)| f.retyped(data_type.clone())),Expr::Placeholder(Placeholder {id: _,field: Some(field),- }) => Ok(field.as_ref().clone().with_name(&schema_name).into()),+ }) => Ok(Arc::clone(field).renamed(&schema_name)),Expr::Like(_)| Expr::SimilarTo(_)| Expr::Not(_)@@ -670,7 +667,8 @@ impl ExprSchemable for Expr {Ok((relation,- Arc::new(field.as_ref().clone().with_name(schema_name)),+ // todo avoid this rename / use the name above+ field.renamed(&schema_name),))}@@ -1172,25 +1170,25 @@ mod tests {#[derive(Debug)]struct MockExprSchema {- field: Field,+ field: FieldRef,error_on_nullable: bool,}impl MockExprSchema {fn new() -> Self {Self {- field: Field::new("mock_field", DataType::Null, false),+ field: Arc::new(Field::new("mock_field", DataType::Null, false)),error_on_nullable: false,}}fn with_nullable(mut self, nullable: bool) -> Self {- self.field = self.field.with_nullable(nullable);+ Arc::make_mut(&mut self.field).set_nullable(nullable);self}fn with_data_type(mut self, data_type: DataType) -> Self {- self.field = self.field.with_data_type(data_type);+ Arc::make_mut(&mut self.field).set_data_type(data_type);self}@@ -1200,7 +1198,8 @@ mod tests {}fn with_metadata(mut self, metadata: FieldMetadata) -> Self {- self.field = metadata.add_to_field(self.field);+ self.field =+ Arc::new(metadata.add_to_field(Arc::unwrap_or_clone(self.field)));self}}@@ -1211,7 +1210,7 @@ mod tests {Ok(self.field.is_nullable())}- fn field_from_column(&self, _col: &Column) -> Result<&Field> {+ fn field_from_column(&self, _col: &Column) -> Result<&FieldRef> {Ok(&self.field)}}diff --git a/datafusion/sql/src/expr/identifier.rs b/datafusion/sql/src/expr/identifier.rsindex b5db580217..b329cc5d1f 100644--- a/datafusion/sql/src/expr/identifier.rs+++ b/datafusion/sql/src/expr/identifier.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use arrow::datatypes::Field;+use arrow::datatypes::FieldRef;use datafusion_common::{assert_or_internal_err, exec_datafusion_err, internal_err, not_impl_err,plan_datafusion_err, plan_err, Column, DFSchema, Result, Span, TableReference,@@ -76,7 +76,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {{// Found an exact match on a qualified name in the outer plan schema, so this is an outer reference columnreturn Ok(Expr::OuterReferenceColumn(- Arc::new(field.clone()),+ Arc::clone(field),Column::from((qualifier, field)),));}@@ -179,7 +179,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {Some((field, qualifier, _nested_names)) => {// Found an exact match on a qualified name in the outer plan schema, so this is an outer reference columnOk(Expr::OuterReferenceColumn(- Arc::new(field.clone()),+ Arc::clone(field),Column::from((qualifier, field)),))}@@ -291,7 +291,7 @@ fn search_dfschema<'ids, 'schema>(ids: &'ids [String],schema: &'schema DFSchema,) -> Option<(- &'schema Field,+ &'schema FieldRef,Option<&'schema TableReference>,&'ids [String],)> {diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rsindex e89ddaa349..a30c6c31fa 100644--- a/datafusion/sql/src/statement.rs+++ b/datafusion/sql/src/statement.rs@@ -2166,7 +2166,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {} else {value_indices[column_index] = Some(i);}- Ok(table_schema.field(column_index).clone())+ Ok(Arc::clone(table_schema.field(column_index)))}).collect::<Result<Vec<_>>>()?;(Fields::from(fields), value_indices)diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rsindex 48e93c04bb..5f2a4bca4b 100644--- a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs+++ b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs@@ -242,9 +242,7 @@ pub fn apply_masking(let fields = column_indices.iter().map(|i| schema.qualified_field(*i))- .map(|(qualifier, field)| {- (qualifier.cloned(), Arc::new(field.clone()))- })+ .map(|(qualifier, field)| (qualifier.cloned(), Arc::clone(field))).collect();Ok(DFSchema::new_with_metadata(@@ -288,7 +286,7 @@ fn apply_projection(let fields = column_indices.iter().map(|i| df_schema.qualified_field(*i))- .map(|(qualifier, field)| (qualifier.cloned(), Arc::new(field.clone())))+ .map(|(qualifier, field)| (qualifier.cloned(), Arc::clone(field))).collect();scan.projected_schema = DFSchemaRef::new(DFSchema::new_with_metadata(diff --git a/datafusion/substrait/src/logical_plan/consumer/utils.rs b/datafusion/substrait/src/logical_plan/consumer/utils.rsindex f7eedcb7a2..993a92212b 100644--- a/datafusion/substrait/src/logical_plan/consumer/utils.rs+++ b/datafusion/substrait/src/logical_plan/consumer/utils.rs@@ -577,12 +577,12 @@ pub(crate) mod tests {assert_eq!(renamed_schema.fields().len(), 5);assert_eq!(- *renamed_schema.field(0),- Field::new("a", DataType::Int32, false)+ renamed_schema.field(0),+ &Arc::new(Field::new("a", DataType::Int32, false)));assert_eq!(- *renamed_schema.field(1),- Field::new_struct(+ renamed_schema.field(1),+ &Arc::new(Field::new_struct("b",vec![Field::new("c", DataType::Int32, false),@@ -593,11 +593,11 @@ pub(crate) mod tests {)],false,- )+ )));assert_eq!(- *renamed_schema.field(2),- Field::new_list(+ renamed_schema.field(2),+ &Arc::new(Field::new_list("f",Arc::new(Field::new_struct("item",@@ -605,11 +605,11 @@ pub(crate) mod tests {false,)),false,- )+ )));assert_eq!(- *renamed_schema.field(3),- Field::new_large_list(+ renamed_schema.field(3),+ &Arc::new(Field::new_large_list("h",Arc::new(Field::new_struct("item",@@ -617,11 +617,11 @@ pub(crate) mod tests {false,)),false,- )+ )));assert_eq!(- *renamed_schema.field(4),- Field::new_map(+ renamed_schema.field(4),+ &Arc::new(Field::new_map("j","entries",Arc::new(Field::new_struct(@@ -636,7 +636,7 @@ pub(crate) mod tests {)),false,false,- )+ )));Ok(())}diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.mdindex 25c209c5eb..d40fee0938 100644--- a/docs/source/library-user-guide/upgrading.md+++ b/docs/source/library-user-guide/upgrading.md@@ -25,6 +25,21 @@You can see the current [status of the `52.0.0`release here](https://github.com/apache/datafusion/issues/18566)+### Changes to DFSchema API++To permit more efficient planning, several methods on `DFSchema` have been+changed to return references to the underlying [`&FieldRef`] rather than+[`&Field`]. This allows planners to more cheaply copy the references via+`Arc::clone` rather than cloning the entire `Field` structure.++You may need to change code to use `Arc::clone` instead of `.as_ref().clone()`+directly on the `Field`. For example:++```diff+- let field = df_schema.field("my_column").as_ref().clone();++ let field = Arc::clone(df_schema.field("my_column"));+```+### Removal of `pyarrow` featureThe `pyarrow` feature flag has been removed. This feature has been migrated to