Apache DataFusion SQL Query Engine
refactor: remove core crate from datafusion-proto (#18123)
Tim Saucerf199b0008613
diff --git a/Cargo.lock b/Cargo.lockindex 7e1d2b332a..13fe25c914 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2533,11 +2533,24 @@ dependencies = ["arrow","chrono","datafusion",+ "datafusion-catalog",+ "datafusion-catalog-listing","datafusion-common",+ "datafusion-datasource",+ "datafusion-datasource-arrow",+ "datafusion-datasource-avro",+ "datafusion-datasource-csv",+ "datafusion-datasource-json",+ "datafusion-datasource-parquet",+ "datafusion-execution","datafusion-expr","datafusion-functions","datafusion-functions-aggregate",+ "datafusion-functions-table","datafusion-functions-window-common",+ "datafusion-physical-expr",+ "datafusion-physical-expr-common",+ "datafusion-physical-plan","datafusion-proto-common","doc-comment","object_store",diff --git a/benchmarks/src/imdb/run.rs b/benchmarks/src/imdb/run.rsindex 3d58d5f54d..11bd424ba6 100644--- a/benchmarks/src/imdb/run.rs+++ b/benchmarks/src/imdb/run.rs@@ -534,7 +534,7 @@ mod tests {let plan = ctx.sql(&query).await?;let plan = plan.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let plan2 = logical_plan_from_bytes(&bytes, &ctx)?;+ let plan2 = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;let plan_formatted = format!("{}", plan.display_indent());let plan2_formatted = format!("{}", plan2.display_indent());assert_eq!(plan_formatted, plan2_formatted);diff --git a/benchmarks/src/tpch/run.rs b/benchmarks/src/tpch/run.rsindex b93bdf254a..2b66de641b 100644--- a/benchmarks/src/tpch/run.rs+++ b/benchmarks/src/tpch/run.rs@@ -387,7 +387,7 @@ mod tests {let plan = ctx.sql(&query).await?;let plan = plan.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let plan2 = logical_plan_from_bytes(&bytes, &ctx)?;+ let plan2 = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;let plan_formatted = format!("{}", plan.display_indent());let plan2_formatted = format!("{}", plan2.display_indent());assert_eq!(plan_formatted, plan2_formatted);diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.tomlindex c1d894a6c0..920e277b8c 100644--- a/datafusion/proto/Cargo.toml+++ b/datafusion/proto/Cargo.toml@@ -40,15 +40,31 @@ name = "datafusion_proto"[features]default = ["parquet"]json = ["pbjson", "serde", "serde_json", "datafusion-proto-common/json"]-parquet = ["datafusion/parquet", "datafusion-common/parquet"]-avro = ["datafusion/avro", "datafusion-common/avro"]+parquet = ["datafusion-datasource-parquet", "datafusion-common/parquet", "datafusion/parquet"]+avro = ["datafusion-datasource-avro", "datafusion-common/avro"]++# Note to developers: do *not* add `datafusion` as a dependency in+# this crate. See https://github.com/apache/datafusion/issues/17713+# for additional information.[dependencies]arrow = { workspace = true }chrono = { workspace = true }-datafusion = { workspace = true, default-features = false }+datafusion-catalog = { workspace = true }+datafusion-catalog-listing = { workspace = true }datafusion-common = { workspace = true }+datafusion-datasource = { workspace = true }+datafusion-datasource-arrow = { workspace = true }+datafusion-datasource-avro = { workspace = true, optional = true }+datafusion-datasource-csv = { workspace = true }+datafusion-datasource-json = { workspace = true }+datafusion-datasource-parquet = { workspace = true, optional = true }+datafusion-execution = { workspace = true }datafusion-expr = { workspace = true }+datafusion-functions-table = { workspace = true }+datafusion-physical-expr = { workspace = true }+datafusion-physical-expr-common = { workspace = true }+datafusion-physical-plan = { workspace = true }datafusion-proto-common = { workspace = true }object_store = { workspace = true }pbjson = { workspace = true, optional = true }diff --git a/datafusion/proto/src/bytes/mod.rs b/datafusion/proto/src/bytes/mod.rsindex 5b07e59e80..12d9938373 100644--- a/datafusion/proto/src/bytes/mod.rs+++ b/datafusion/proto/src/bytes/mod.rs@@ -24,8 +24,8 @@ use crate::physical_plan::{AsExecutionPlan, DefaultPhysicalExtensionCodec, PhysicalExtensionCodec,};use crate::protobuf;-use datafusion::execution::TaskContext;use datafusion_common::{plan_datafusion_err, Result};+use datafusion_execution::TaskContext;use datafusion_expr::{create_udaf, create_udf, create_udwf, AggregateUDF, Expr, LogicalPlan, Volatility,WindowUDF,@@ -37,10 +37,9 @@ use prost::{use std::sync::Arc;// Reexport Bytes which appears in the API-use datafusion::execution::registry::FunctionRegistry;-use datafusion::physical_plan::ExecutionPlan;-use datafusion::prelude::SessionContext;+use datafusion_execution::registry::FunctionRegistry;use datafusion_expr::planner::ExprPlanner;+use datafusion_physical_plan::ExecutionPlan;mod registry;@@ -240,16 +239,13 @@ pub fn logical_plan_to_json_with_extension_codec(/// Deserialize a LogicalPlan from JSON#[cfg(feature = "json")]-pub fn logical_plan_from_json(json: &str, ctx: &SessionContext) -> Result<LogicalPlan> {+pub fn logical_plan_from_json(json: &str, ctx: &TaskContext) -> Result<LogicalPlan> {let extension_codec = DefaultLogicalExtensionCodec {};logical_plan_from_json_with_extension_codec(json, ctx, &extension_codec)}/// Deserialize a LogicalPlan from bytes-pub fn logical_plan_from_bytes(- bytes: &[u8],- ctx: &SessionContext,-) -> Result<LogicalPlan> {+pub fn logical_plan_from_bytes(bytes: &[u8], ctx: &TaskContext) -> Result<LogicalPlan> {let extension_codec = DefaultLogicalExtensionCodec {};logical_plan_from_bytes_with_extension_codec(bytes, ctx, &extension_codec)}@@ -257,7 +253,7 @@ pub fn logical_plan_from_bytes(/// Deserialize a LogicalPlan from bytespub fn logical_plan_from_bytes_with_extension_codec(bytes: &[u8],- ctx: &SessionContext,+ ctx: &TaskContext,extension_codec: &dyn LogicalExtensionCodec,) -> Result<LogicalPlan> {let protobuf = protobuf::LogicalPlanNode::decode(bytes)@@ -269,7 +265,7 @@ pub fn logical_plan_from_bytes_with_extension_codec(#[cfg(feature = "json")]pub fn logical_plan_from_json_with_extension_codec(json: &str,- ctx: &SessionContext,+ ctx: &TaskContext,extension_codec: &dyn LogicalExtensionCodec,) -> Result<LogicalPlan> {let back: protobuf::LogicalPlanNode = serde_json::from_str(json)@@ -312,12 +308,12 @@ pub fn physical_plan_to_bytes_with_extension_codec(#[cfg(feature = "json")]pub fn physical_plan_from_json(json: &str,- ctx: &SessionContext,+ ctx: &TaskContext,) -> Result<Arc<dyn ExecutionPlan>> {let back: protobuf::PhysicalPlanNode = serde_json::from_str(json).map_err(|e| plan_datafusion_err!("Error serializing plan: {e}"))?;let extension_codec = DefaultPhysicalExtensionCodec {};- back.try_into_physical_plan(&ctx.task_ctx(), &extension_codec)+ back.try_into_physical_plan(&ctx, &extension_codec)}/// Deserialize a PhysicalPlan from bytesdiff --git a/datafusion/proto/src/bytes/registry.rs b/datafusion/proto/src/bytes/registry.rsindex 5d46d41f79..087e073db2 100644--- a/datafusion/proto/src/bytes/registry.rs+++ b/datafusion/proto/src/bytes/registry.rs@@ -17,9 +17,9 @@use std::{collections::HashSet, sync::Arc};-use datafusion::execution::registry::FunctionRegistry;use datafusion_common::plan_err;use datafusion_common::Result;+use datafusion_execution::registry::FunctionRegistry;use datafusion_expr::planner::ExprPlanner;use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};diff --git a/datafusion/proto/src/lib.rs b/datafusion/proto/src/lib.rsindex bb7b992f14..b1590b9ad2 100644--- a/datafusion/proto/src/lib.rs+++ b/datafusion/proto/src/lib.rs@@ -34,8 +34,8 @@//!//! [`LogicalPlan`]: datafusion_expr::LogicalPlan//! [`Expr`]: datafusion_expr::Expr-//! [`ExecutionPlan`]: datafusion::physical_plan::ExecutionPlan-//! [`PhysicalExpr`]: datafusion::physical_expr::PhysicalExpr+//! [`ExecutionPlan`]: datafusion_physical_plan::ExecutionPlan+//! [`PhysicalExpr`]: datafusion_physical_expr::PhysicalExpr//!//! Internally, this crate is implemented by converting the plans to [protocol//! buffers] using [prost].@@ -93,7 +93,7 @@//! let bytes = logical_plan_to_bytes(&plan)?;//!//! // Decode bytes from somewhere (over network, etc.) back to LogicalPlan-//! let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+//! let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;//! assert_eq!(format!("{:?}", plan), format!("{:?}", logical_round_trip));//! # Ok(())//! # }diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rsindex 0e76e19ecb..d32bfb22ff 100644--- a/datafusion/proto/src/logical_plan/file_formats.rs+++ b/datafusion/proto/src/logical_plan/file_formats.rs@@ -17,22 +17,19 @@use std::sync::Arc;-use datafusion::{- config::{CsvOptions, JsonOptions},- datasource::file_format::{- arrow::ArrowFormatFactory, csv::CsvFormatFactory, json::JsonFormatFactory,- FileFormatFactory,- },- prelude::SessionContext,-};+use crate::protobuf::{CsvOptions as CsvOptionsProto, JsonOptions as JsonOptionsProto};+use datafusion_common::config::{CsvOptions, JsonOptions};use datafusion_common::{exec_datafusion_err, exec_err, not_impl_err, parsers::CompressionTypeVariant,TableReference,};+use datafusion_datasource::file_format::FileFormatFactory;+use datafusion_datasource_arrow::file_format::ArrowFormatFactory;+use datafusion_datasource_csv::file_format::CsvFormatFactory;+use datafusion_datasource_json::file_format::JsonFormatFactory;+use datafusion_execution::TaskContext;use prost::Message;-use crate::protobuf::{CsvOptions as CsvOptionsProto, JsonOptions as JsonOptionsProto};-use super::LogicalExtensionCodec;#[derive(Debug)]@@ -165,7 +162,7 @@ impl LogicalExtensionCodec for CsvLogicalExtensionCodec {&self,_buf: &[u8],_inputs: &[datafusion_expr::LogicalPlan],- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> datafusion_common::Result<datafusion_expr::Extension> {not_impl_err!("Method not implemented")}@@ -183,15 +180,15 @@ impl LogicalExtensionCodec for CsvLogicalExtensionCodec {_buf: &[u8],_table_ref: &TableReference,_schema: arrow::datatypes::SchemaRef,- _ctx: &SessionContext,- ) -> datafusion_common::Result<Arc<dyn datafusion::datasource::TableProvider>> {+ _ctx: &TaskContext,+ ) -> datafusion_common::Result<Arc<dyn datafusion_catalog::TableProvider>> {not_impl_err!("Method not implemented")}fn try_encode_table_provider(&self,_table_ref: &TableReference,- _node: Arc<dyn datafusion::datasource::TableProvider>,+ _node: Arc<dyn datafusion_catalog::TableProvider>,_buf: &mut Vec<u8>,) -> datafusion_common::Result<()> {not_impl_err!("Method not implemented")@@ -200,7 +197,7 @@ impl LogicalExtensionCodec for CsvLogicalExtensionCodec {fn try_decode_file_format(&self,buf: &[u8],- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> datafusion_common::Result<Arc<dyn FileFormatFactory>> {let proto = CsvOptionsProto::decode(buf).map_err(|e| {exec_datafusion_err!("Failed to decode CsvOptionsProto: {e:?}")@@ -272,7 +269,7 @@ impl LogicalExtensionCodec for JsonLogicalExtensionCodec {&self,_buf: &[u8],_inputs: &[datafusion_expr::LogicalPlan],- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> datafusion_common::Result<datafusion_expr::Extension> {not_impl_err!("Method not implemented")}@@ -290,15 +287,15 @@ impl LogicalExtensionCodec for JsonLogicalExtensionCodec {_buf: &[u8],_table_ref: &TableReference,_schema: arrow::datatypes::SchemaRef,- _ctx: &SessionContext,- ) -> datafusion_common::Result<Arc<dyn datafusion::datasource::TableProvider>> {+ _ctx: &TaskContext,+ ) -> datafusion_common::Result<Arc<dyn datafusion_catalog::TableProvider>> {not_impl_err!("Method not implemented")}fn try_encode_table_provider(&self,_table_ref: &TableReference,- _node: Arc<dyn datafusion::datasource::TableProvider>,+ _node: Arc<dyn datafusion_catalog::TableProvider>,_buf: &mut Vec<u8>,) -> datafusion_common::Result<()> {not_impl_err!("Method not implemented")@@ -307,7 +304,7 @@ impl LogicalExtensionCodec for JsonLogicalExtensionCodec {fn try_decode_file_format(&self,buf: &[u8],- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> datafusion_common::Result<Arc<dyn FileFormatFactory>> {let proto = JsonOptionsProto::decode(buf).map_err(|e| {exec_datafusion_err!("Failed to decode JsonOptionsProto: {e:?}")@@ -353,11 +350,10 @@ mod parquet {ParquetOptions as ParquetOptionsProto,TableParquetOptions as TableParquetOptionsProto,};-- use datafusion::{- config::{ParquetColumnOptions, ParquetOptions, TableParquetOptions},- datasource::file_format::parquet::ParquetFormatFactory,+ use datafusion_common::config::{+ ParquetColumnOptions, ParquetOptions, TableParquetOptions,};+ use datafusion_datasource_parquet::file_format::ParquetFormatFactory;impl TableParquetOptionsProto {fn from_factory(factory: &ParquetFormatFactory) -> Self {@@ -595,7 +591,7 @@ mod parquet {&self,_buf: &[u8],_inputs: &[datafusion_expr::LogicalPlan],- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> datafusion_common::Result<datafusion_expr::Extension> {not_impl_err!("Method not implemented")}@@ -613,8 +609,8 @@ mod parquet {_buf: &[u8],_table_ref: &TableReference,_schema: arrow::datatypes::SchemaRef,- _ctx: &SessionContext,- ) -> datafusion_common::Result<Arc<dyn datafusion::datasource::TableProvider>>+ _ctx: &TaskContext,+ ) -> datafusion_common::Result<Arc<dyn datafusion_catalog::TableProvider>>{not_impl_err!("Method not implemented")}@@ -622,7 +618,7 @@ mod parquet {fn try_encode_table_provider(&self,_table_ref: &TableReference,- _node: Arc<dyn datafusion::datasource::TableProvider>,+ _node: Arc<dyn datafusion_catalog::TableProvider>,_buf: &mut Vec<u8>,) -> datafusion_common::Result<()> {not_impl_err!("Method not implemented")@@ -631,14 +627,14 @@ mod parquet {fn try_decode_file_format(&self,buf: &[u8],- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> datafusion_common::Result<Arc<dyn FileFormatFactory>> {let proto = TableParquetOptionsProto::decode(buf).map_err(|e| {exec_datafusion_err!("Failed to decode TableParquetOptionsProto: {e:?}")})?;let options: TableParquetOptions = (&proto).into();Ok(Arc::new(- datafusion::datasource::file_format::parquet::ParquetFormatFactory {+ datafusion_datasource_parquet::file_format::ParquetFormatFactory {options: Some(options),},))@@ -649,7 +645,7 @@ mod parquet {buf: &mut Vec<u8>,node: Arc<dyn FileFormatFactory>,) -> datafusion_common::Result<()> {- use datafusion::datasource::file_format::parquet::ParquetFormatFactory;+ use datafusion_datasource_parquet::file_format::ParquetFormatFactory;let options = if let Some(parquet_factory) =node.as_any().downcast_ref::<ParquetFormatFactory>()@@ -683,7 +679,7 @@ impl LogicalExtensionCodec for ArrowLogicalExtensionCodec {&self,_buf: &[u8],_inputs: &[datafusion_expr::LogicalPlan],- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> datafusion_common::Result<datafusion_expr::Extension> {not_impl_err!("Method not implemented")}@@ -701,15 +697,15 @@ impl LogicalExtensionCodec for ArrowLogicalExtensionCodec {_buf: &[u8],_table_ref: &TableReference,_schema: arrow::datatypes::SchemaRef,- _ctx: &SessionContext,- ) -> datafusion_common::Result<Arc<dyn datafusion::datasource::TableProvider>> {+ _ctx: &TaskContext,+ ) -> datafusion_common::Result<Arc<dyn datafusion_catalog::TableProvider>> {not_impl_err!("Method not implemented")}fn try_encode_table_provider(&self,_table_ref: &TableReference,- _node: Arc<dyn datafusion::datasource::TableProvider>,+ _node: Arc<dyn datafusion_catalog::TableProvider>,_buf: &mut Vec<u8>,) -> datafusion_common::Result<()> {not_impl_err!("Method not implemented")@@ -718,7 +714,7 @@ impl LogicalExtensionCodec for ArrowLogicalExtensionCodec {fn try_decode_file_format(&self,__buf: &[u8],- __ctx: &SessionContext,+ __ctx: &TaskContext,) -> datafusion_common::Result<Arc<dyn FileFormatFactory>> {Ok(Arc::new(ArrowFormatFactory::new()))}@@ -741,7 +737,7 @@ impl LogicalExtensionCodec for AvroLogicalExtensionCodec {&self,_buf: &[u8],_inputs: &[datafusion_expr::LogicalPlan],- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> datafusion_common::Result<datafusion_expr::Extension> {not_impl_err!("Method not implemented")}@@ -759,15 +755,15 @@ impl LogicalExtensionCodec for AvroLogicalExtensionCodec {_buf: &[u8],_table_ref: &TableReference,_schema: arrow::datatypes::SchemaRef,- _cts: &SessionContext,- ) -> datafusion_common::Result<Arc<dyn datafusion::datasource::TableProvider>> {+ _cts: &TaskContext,+ ) -> datafusion_common::Result<Arc<dyn datafusion_catalog::TableProvider>> {not_impl_err!("Method not implemented")}fn try_encode_table_provider(&self,_table_ref: &TableReference,- _node: Arc<dyn datafusion::datasource::TableProvider>,+ _node: Arc<dyn datafusion_catalog::TableProvider>,_buf: &mut Vec<u8>,) -> datafusion_common::Result<()> {not_impl_err!("Method not implemented")@@ -776,7 +772,7 @@ impl LogicalExtensionCodec for AvroLogicalExtensionCodec {fn try_decode_file_format(&self,__buf: &[u8],- __ctx: &SessionContext,+ __ctx: &TaskContext,) -> datafusion_common::Result<Arc<dyn FileFormatFactory>> {Ok(Arc::new(ArrowFormatFactory::new()))}diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rsindex ec6415adc4..4296867049 100644--- a/datafusion/proto/src/logical_plan/from_proto.rs+++ b/datafusion/proto/src/logical_plan/from_proto.rs@@ -17,11 +17,11 @@use std::sync::Arc;-use datafusion::execution::registry::FunctionRegistry;use datafusion_common::{exec_datafusion_err, internal_err, plan_datafusion_err, NullEquality,RecursionUnnestOption, Result, ScalarValue, TableReference, UnnestOptions,};+use datafusion_execution::registry::FunctionRegistry;use datafusion_expr::dml::InsertOp;use datafusion_expr::expr::{Alias, NullTreatment, Placeholder, Sort};use datafusion_expr::expr::{Unnest, WildcardOptions};diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex fd9e07914b..ad56185166 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -34,32 +34,23 @@ use crate::{use crate::protobuf::{proto_error, ToProtoError};use arrow::datatypes::{DataType, Schema, SchemaBuilder, SchemaRef};-use datafusion::datasource::cte_worktable::CteWorkTable;-use datafusion::datasource::file_format::arrow::ArrowFormat;-#[cfg(feature = "avro")]-use datafusion::datasource::file_format::avro::AvroFormat;-#[cfg(feature = "parquet")]-use datafusion::datasource::file_format::parquet::ParquetFormat;-use datafusion::datasource::file_format::{- file_type_to_format, format_as_file_type, FileFormatFactory,-};-use datafusion::{- datasource::{- file_format::{- csv::CsvFormat, json::JsonFormat as OtherNdJsonFormat, FileFormat,- },- listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl},- view::ViewTable,- TableProvider,- },- datasource::{provider_as_source, source_as_provider},- prelude::SessionContext,-};+use datafusion_catalog::cte_worktable::CteWorkTable;use datafusion_common::file_options::file_type::FileType;use datafusion_common::{context, internal_datafusion_err, internal_err, not_impl_err, plan_err, Result,TableReference, ToDFSchema,};+use datafusion_datasource::file_format::FileFormat;+use datafusion_datasource::file_format::{+ file_type_to_format, format_as_file_type, FileFormatFactory,+};+use datafusion_datasource_arrow::file_format::ArrowFormat;+#[cfg(feature = "avro")]+use datafusion_datasource_avro::file_format::AvroFormat;+use datafusion_datasource_csv::file_format::CsvFormat;+use datafusion_datasource_json::file_format::JsonFormat as OtherNdJsonFormat;+#[cfg(feature = "parquet")]+use datafusion_datasource_parquet::file_format::ParquetFormat;use datafusion_expr::{dml,logical_plan::{@@ -77,6 +68,12 @@ use datafusion_expr::{use self::to_proto::{serialize_expr, serialize_exprs};use crate::logical_plan::to_proto::serialize_sorts;+use datafusion_catalog::default_table_source::{provider_as_source, source_as_provider};+use datafusion_catalog::view::ViewTable;+use datafusion_catalog::TableProvider;+use datafusion_catalog_listing::{ListingOptions, ListingTable, ListingTableConfig};+use datafusion_datasource::ListingTableUrl;+use datafusion_execution::TaskContext;use prost::bytes::BufMut;use prost::Message;@@ -96,7 +93,7 @@ pub trait AsLogicalPlan: Debug + Send + Sync + Clone {fn try_into_logical_plan(&self,- ctx: &SessionContext,+ ctx: &TaskContext,extension_codec: &dyn LogicalExtensionCodec,) -> Result<LogicalPlan>;@@ -113,7 +110,7 @@ pub trait LogicalExtensionCodec: Debug + Send + Sync {&self,buf: &[u8],inputs: &[LogicalPlan],- ctx: &SessionContext,+ ctx: &TaskContext,) -> Result<Extension>;fn try_encode(&self, node: &Extension, buf: &mut Vec<u8>) -> Result<()>;@@ -123,7 +120,7 @@ pub trait LogicalExtensionCodec: Debug + Send + Sync {buf: &[u8],table_ref: &TableReference,schema: SchemaRef,- ctx: &SessionContext,+ ctx: &TaskContext,) -> Result<Arc<dyn TableProvider>>;fn try_encode_table_provider(@@ -136,7 +133,7 @@ pub trait LogicalExtensionCodec: Debug + Send + Sync {fn try_decode_file_format(&self,_buf: &[u8],- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> Result<Arc<dyn FileFormatFactory>> {not_impl_err!("LogicalExtensionCodec is not provided for file format")}@@ -184,7 +181,7 @@ impl LogicalExtensionCodec for DefaultLogicalExtensionCodec {&self,_buf: &[u8],_inputs: &[LogicalPlan],- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> Result<Extension> {not_impl_err!("LogicalExtensionCodec is not provided")}@@ -198,7 +195,7 @@ impl LogicalExtensionCodec for DefaultLogicalExtensionCodec {_buf: &[u8],_table_ref: &TableReference,_schema: SchemaRef,- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> Result<Arc<dyn TableProvider>> {not_impl_err!("LogicalExtensionCodec is not provided")}@@ -242,7 +239,7 @@ fn from_table_reference(/// serialized by [from_table_source]fn to_table_source(node: &Option<Box<LogicalPlanNode>>,- ctx: &SessionContext,+ ctx: &TaskContext,extension_codec: &dyn LogicalExtensionCodec,) -> Result<Arc<dyn TableSource>> {if let Some(node) = node {@@ -296,7 +293,7 @@ impl AsLogicalPlan for LogicalPlanNode {fn try_into_logical_plan(&self,- ctx: &SessionContext,+ ctx: &TaskContext,extension_codec: &dyn LogicalExtensionCodec,) -> Result<LogicalPlan> {let plan = self.logical_plan_type.as_ref().ok_or_else(|| {@@ -475,10 +472,7 @@ impl AsLogicalPlan for LogicalPlanNode {.with_schema(Arc::new(schema));let provider = ListingTable::try_new(config)?.with_cache(- ctx.state()- .runtime_env()- .cache_manager- .get_file_statistic_cache(),+ ctx.runtime_env().cache_manager.get_file_statistic_cache(),);let table_name =@@ -547,7 +541,7 @@ impl AsLogicalPlan for LogicalPlanNode {.build()}LogicalPlanType::Repartition(repartition) => {- use datafusion::logical_expr::Partitioning;+ use datafusion_expr::Partitioning;let input: LogicalPlan =into_logical_plan!(repartition.input, ctx, extension_codec)?;use protobuf::repartition_node::PartitionMethod;@@ -595,11 +589,6 @@ impl AsLogicalPlan for LogicalPlanNode {None};- let file_type = create_extern_table.file_type.as_str();- if ctx.table_factory(file_type).is_none() {- internal_err!("No TableProviderFactory for file type: {file_type}")?- }-let mut order_exprs = vec![];for expr in &create_extern_table.order_exprs {order_exprs.push(from_proto::parse_sorts(@@ -961,14 +950,14 @@ impl AsLogicalPlan for LogicalPlanNode {)?.build()}- LogicalPlanType::Dml(dml_node) => Ok(LogicalPlan::Dml(- datafusion::logical_expr::DmlStatement::new(+ LogicalPlanType::Dml(dml_node) => {+ Ok(LogicalPlan::Dml(datafusion_expr::DmlStatement::new(from_table_reference(dml_node.table_name.as_ref(), "DML ")?,to_table_source(&dml_node.target, ctx, extension_codec)?,dml_node.dml_type().into(),Arc::new(into_logical_plan!(dml_node.input, ctx, extension_codec)?),- ),- )),+ )))+ }}}@@ -1417,7 +1406,7 @@ impl AsLogicalPlan for LogicalPlanNode {input,partitioning_scheme,}) => {- use datafusion::logical_expr::Partitioning;+ use datafusion_expr::Partitioning;let input: LogicalPlanNode = LogicalPlanNode::try_from_logical_plan(input.as_ref(),extension_codec,diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex e2ee1be7d7..7c4b9e55b8 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -28,26 +28,27 @@ use datafusion_expr::dml::InsertOp;use object_store::path::Path;use object_store::ObjectMeta;-use datafusion::arrow::datatypes::Schema;-use datafusion::datasource::file_format::csv::CsvSink;-use datafusion::datasource::file_format::json::JsonSink;+use arrow::datatypes::Schema;+use datafusion_common::{internal_datafusion_err, not_impl_err, DataFusionError, Result};+use datafusion_datasource::file::FileSource;+use datafusion_datasource::file_groups::FileGroup;+use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};+use datafusion_datasource::file_sink_config::FileSinkConfig;+use datafusion_datasource::{FileRange, ListingTableUrl, PartitionedFile};+use datafusion_datasource_csv::file_format::CsvSink;+use datafusion_datasource_json::file_format::JsonSink;#[cfg(feature = "parquet")]-use datafusion::datasource::file_format::parquet::ParquetSink;-use datafusion::datasource::listing::{FileRange, ListingTableUrl, PartitionedFile};-use datafusion::datasource::object_store::ObjectStoreUrl;-use datafusion::datasource::physical_plan::{- FileGroup, FileScanConfig, FileScanConfigBuilder, FileSinkConfig, FileSource,-};-use datafusion::execution::{FunctionRegistry, TaskContext};-use datafusion::logical_expr::WindowFunctionDefinition;-use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr, ScalarFunctionExpr};-use datafusion::physical_plan::expressions::{+use datafusion_datasource_parquet::file_format::ParquetSink;+use datafusion_execution::object_store::ObjectStoreUrl;+use datafusion_execution::{FunctionRegistry, TaskContext};+use datafusion_expr::WindowFunctionDefinition;+use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr, ScalarFunctionExpr};+use datafusion_physical_plan::expressions::{in_list, BinaryExpr, CaseExpr, CastExpr, Column, IsNotNullExpr, IsNullExpr, LikeExpr,Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,};-use datafusion::physical_plan::windows::{create_window_expr, schema_add_window_field};-use datafusion::physical_plan::{Partitioning, PhysicalExpr, WindowExpr};-use datafusion_common::{internal_datafusion_err, not_impl_err, DataFusionError, Result};+use datafusion_physical_plan::windows::{create_window_expr, schema_add_window_field};+use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};use datafusion_proto_common::common::proto_error;use crate::convert_required;diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex 818109be68..cd9bd67285 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -23,8 +23,7 @@ use self::to_proto::{serialize_partitioning, serialize_physical_expr};use crate::common::{byte_to_string, str_to_byte};use crate::physical_plan::from_proto::{parse_physical_expr, parse_physical_sort_expr, parse_physical_sort_exprs,- parse_physical_window_expr, parse_protobuf_file_scan_config,- parse_protobuf_file_scan_schema, parse_record_batches,+ parse_physical_window_expr, parse_protobuf_file_scan_config, parse_record_batches,};use crate::physical_plan::to_proto::{serialize_file_scan_config, serialize_maybe_filter, serialize_physical_aggr_expr,@@ -40,65 +39,64 @@ use crate::protobuf::{};use crate::{convert_required, into_required};-use datafusion::arrow::compute::SortOptions;-use datafusion::arrow::datatypes::{IntervalMonthDayNanoType, Schema, SchemaRef};-use datafusion::catalog::memory::MemorySourceConfig;-use datafusion::datasource::file_format::csv::CsvSink;-use datafusion::datasource::file_format::file_compression_type::FileCompressionType;-use datafusion::datasource::file_format::json::JsonSink;+use arrow::compute::SortOptions;+use arrow::datatypes::{IntervalMonthDayNanoType, SchemaRef};+use datafusion_catalog::memory::MemorySourceConfig;+use datafusion_common::{+ internal_datafusion_err, internal_err, not_impl_err, DataFusionError, Result,+};#[cfg(feature = "parquet")]-use datafusion::datasource::file_format::parquet::ParquetSink;+use datafusion_datasource::file::FileSource;+use datafusion_datasource::file_compression_type::FileCompressionType;+use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};+use datafusion_datasource::sink::DataSinkExec;+use datafusion_datasource::source::{DataSource, DataSourceExec};#[cfg(feature = "avro")]-use datafusion::datasource::physical_plan::AvroSource;+use datafusion_datasource_avro::source::AvroSource;+use datafusion_datasource_csv::file_format::CsvSink;+use datafusion_datasource_csv::source::CsvSource;+use datafusion_datasource_json::file_format::JsonSink;+use datafusion_datasource_json::source::JsonSource;#[cfg(feature = "parquet")]-use datafusion::datasource::physical_plan::ParquetSource;-use datafusion::datasource::physical_plan::{- CsvSource, FileScanConfig, FileScanConfigBuilder, FileSource, JsonSource,-};-use datafusion::datasource::sink::DataSinkExec;-use datafusion::datasource::source::{DataSource, DataSourceExec};-use datafusion::execution::{FunctionRegistry, TaskContext};-use datafusion::functions_table::generate_series::{+use datafusion_datasource_parquet::file_format::ParquetSink;+#[cfg(feature = "parquet")]+use datafusion_datasource_parquet::source::ParquetSource;+use datafusion_execution::{FunctionRegistry, TaskContext};+use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};+use datafusion_functions_table::generate_series::{Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue,};-use datafusion::physical_expr::aggregate::AggregateExprBuilder;-use datafusion::physical_expr::aggregate::AggregateFunctionExpr;-use datafusion::physical_expr::{LexOrdering, LexRequirement, PhysicalExprRef};-use datafusion::physical_plan::aggregates::AggregateMode;-use datafusion::physical_plan::aggregates::{AggregateExec, PhysicalGroupBy};-use datafusion::physical_plan::analyze::AnalyzeExec;-use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;-use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;-use datafusion::physical_plan::coop::CooperativeExec;-use datafusion::physical_plan::empty::EmptyExec;-use datafusion::physical_plan::explain::ExplainExec;-use datafusion::physical_plan::expressions::PhysicalSortExpr;-use datafusion::physical_plan::filter::FilterExec;-use datafusion::physical_plan::joins::utils::{ColumnIndex, JoinFilter};-use datafusion::physical_plan::joins::{+use datafusion_physical_expr::aggregate::AggregateExprBuilder;+use datafusion_physical_expr::aggregate::AggregateFunctionExpr;+use datafusion_physical_expr::{LexOrdering, LexRequirement, PhysicalExprRef};+use datafusion_physical_plan::aggregates::AggregateMode;+use datafusion_physical_plan::aggregates::{AggregateExec, PhysicalGroupBy};+use datafusion_physical_plan::analyze::AnalyzeExec;+use datafusion_physical_plan::coalesce_batches::CoalesceBatchesExec;+use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;+use datafusion_physical_plan::coop::CooperativeExec;+use datafusion_physical_plan::empty::EmptyExec;+use datafusion_physical_plan::explain::ExplainExec;+use datafusion_physical_plan::expressions::PhysicalSortExpr;+use datafusion_physical_plan::filter::FilterExec;+use datafusion_physical_plan::joins::utils::{ColumnIndex, JoinFilter};+use datafusion_physical_plan::joins::{CrossJoinExec, NestedLoopJoinExec, SortMergeJoinExec, StreamJoinPartitionMode,SymmetricHashJoinExec,};-use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode};-use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};-use datafusion::physical_plan::memory::LazyMemoryExec;-use datafusion::physical_plan::metrics::MetricType;-use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;-use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};-use datafusion::physical_plan::repartition::RepartitionExec;-use datafusion::physical_plan::sorts::sort::SortExec;-use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;-use datafusion::physical_plan::union::{InterleaveExec, UnionExec};-use datafusion::physical_plan::unnest::{ListUnnest, UnnestExec};-use datafusion::physical_plan::windows::{BoundedWindowAggExec, WindowAggExec};-use datafusion::physical_plan::{- ExecutionPlan, InputOrderMode, PhysicalExpr, WindowExpr,-};-use datafusion_common::config::TableParquetOptions;-use datafusion_common::{- internal_datafusion_err, internal_err, not_impl_err, DataFusionError, Result,-};-use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};+use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};+use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};+use datafusion_physical_plan::memory::LazyMemoryExec;+use datafusion_physical_plan::metrics::MetricType;+use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;+use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};+use datafusion_physical_plan::repartition::RepartitionExec;+use datafusion_physical_plan::sorts::sort::SortExec;+use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;+use datafusion_physical_plan::union::{InterleaveExec, UnionExec};+use datafusion_physical_plan::unnest::{ListUnnest, UnnestExec};+use datafusion_physical_plan::windows::{BoundedWindowAggExec, WindowAggExec};+use datafusion_physical_plan::{ExecutionPlan, InputOrderMode, PhysicalExpr, WindowExpr};use prost::bytes::BufMut;use prost::Message;@@ -662,8 +660,9 @@ impl protobuf::PhysicalPlanNode {) -> Result<Arc<dyn ExecutionPlan>> {#[cfg(feature = "parquet")]{- let schema =- parse_protobuf_file_scan_schema(scan.base_conf.as_ref().unwrap())?;+ let schema = from_proto::parse_protobuf_file_scan_schema(+ scan.base_conf.as_ref().unwrap(),+ )?;// Check if there's a projection and use projected schema for predicate parsinglet base_conf = scan.base_conf.as_ref().unwrap();@@ -674,7 +673,7 @@ impl protobuf::PhysicalPlanNode {.iter().map(|&i| schema.field(i as usize).clone()).collect();- Arc::new(Schema::new(projected_fields))+ Arc::new(arrow::datatypes::Schema::new(projected_fields))} else {schema};@@ -691,7 +690,7 @@ impl protobuf::PhysicalPlanNode {)}).transpose()?;- let mut options = TableParquetOptions::default();+ let mut options = datafusion_common::config::TableParquetOptions::default();if let Some(table_options) = scan.parquet_options.as_ref() {options = table_options.try_into()?;@@ -1694,6 +1693,7 @@ impl protobuf::PhysicalPlanNode {)))}+ #[cfg_attr(not(feature = "parquet"), expect(unused_variables))]fn try_into_parquet_sink_physical_plan(&self,sink: &protobuf::ParquetSinkExecNode,diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex 19a76de3e5..7e29a1deae 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -20,32 +20,30 @@ use std::sync::Arc;use arrow::array::RecordBatch;use arrow::datatypes::Schema;use arrow::ipc::writer::StreamWriter;-#[cfg(feature = "parquet")]-use datafusion::datasource::file_format::parquet::ParquetSink;-use datafusion::datasource::physical_plan::FileSink;-use datafusion::physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};-use datafusion::physical_expr::ScalarFunctionExpr;-use datafusion::physical_expr_common::physical_expr::snapshot_physical_expr;-use datafusion::physical_expr_common::sort_expr::PhysicalSortExpr;-use datafusion::physical_plan::expressions::{- BinaryExpr, CaseExpr, CastExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr,- Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,-};-use datafusion::physical_plan::udaf::AggregateFunctionExpr;-use datafusion::physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};-use datafusion::physical_plan::{Partitioning, PhysicalExpr, WindowExpr};-use datafusion::{- datasource::{- file_format::{csv::CsvSink, json::JsonSink},- listing::{FileRange, PartitionedFile},- physical_plan::{FileScanConfig, FileSinkConfig},- },- physical_plan::expressions::LikeExpr,-};use datafusion_common::{internal_datafusion_err, internal_err, not_impl_err, DataFusionError, Result,};+use datafusion_datasource::file_scan_config::FileScanConfig;+use datafusion_datasource::file_sink_config::FileSink;+use datafusion_datasource::file_sink_config::FileSinkConfig;+use datafusion_datasource::{FileRange, PartitionedFile};+use datafusion_datasource_csv::file_format::CsvSink;+use datafusion_datasource_json::file_format::JsonSink;+#[cfg(feature = "parquet")]+use datafusion_datasource_parquet::file_format::ParquetSink;use datafusion_expr::WindowFrame;+use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};+use datafusion_physical_expr::ScalarFunctionExpr;+use datafusion_physical_expr_common::physical_expr::snapshot_physical_expr;+use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;+use datafusion_physical_plan::expressions::LikeExpr;+use datafusion_physical_plan::expressions::{+ BinaryExpr, CaseExpr, CastExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr,+ Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,+};+use datafusion_physical_plan::udaf::AggregateFunctionExpr;+use datafusion_physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};+use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};use crate::protobuf::{self, physical_aggregate_expr_node, physical_window_expr_node, PhysicalSortExprNode,diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex 3d51038eba..516f178cc0 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -71,6 +71,7 @@ use datafusion_common::{internal_datafusion_err, internal_err, not_impl_err, plan_err, DFSchema, DFSchemaRef,DataFusionError, Result, ScalarValue, TableReference,};+use datafusion_execution::TaskContext;use datafusion_expr::dml::CopyTo;use datafusion_expr::expr::{self, Between, BinaryExpr, Case, Cast, GroupingSet, InList, Like, NullTreatment,@@ -153,8 +154,11 @@ async fn roundtrip_logical_plan() -> Result<()> {});let extension_codec = TopKExtensionCodec {};let bytes = logical_plan_to_bytes_with_extension_codec(&topk_plan, &extension_codec)?;- let logical_round_trip =- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &extension_codec)?;+ let logical_round_trip = logical_plan_from_bytes_with_extension_codec(+ &bytes,+ &ctx.task_ctx(),+ &extension_codec,+ )?;assert_eq!(format!("{topk_plan:?}"), format!("{logical_round_trip:?}"));Ok(())}@@ -177,7 +181,7 @@ impl LogicalExtensionCodec for TestTableProviderCodec {&self,_buf: &[u8],_inputs: &[LogicalPlan],- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> Result<Extension> {not_impl_err!("No extension codec provided")}@@ -191,7 +195,7 @@ impl LogicalExtensionCodec for TestTableProviderCodec {buf: &[u8],table_ref: &TableReference,schema: SchemaRef,- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> Result<Arc<dyn TableProvider>> {let msg = TestTableProto::decode(buf).map_err(|_| internal_datafusion_err!("Error decoding test table"))?;@@ -240,7 +244,7 @@ async fn roundtrip_custom_tables() -> Result<()> {let scan = ctx.table("t").await?.into_optimized_plan()?;let bytes = logical_plan_to_bytes_with_extension_codec(&scan, &codec)?;let logical_round_trip =- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;assert_eq!(format!("{scan:?}"), format!("{logical_round_trip:?}"));Ok(())}@@ -266,7 +270,7 @@ async fn roundtrip_custom_memory_tables() -> Result<()> {let plan = ctx.sql(query).await?.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));Ok(())@@ -293,7 +297,7 @@ async fn roundtrip_custom_listing_tables() -> Result<()> {let plan = ctx.state().create_logical_plan(query).await?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;// Use exact matching to verify everything. Make sure during round-trip,// information like constraints, column defaults, and other aspects of the plan are preserved.assert_eq!(plan, logical_round_trip);@@ -328,7 +332,7 @@ async fn roundtrip_logical_plan_aggregation_with_pk() -> Result<()> {let plan = ctx.sql(query).await?.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));Ok(())@@ -354,7 +358,7 @@ async fn roundtrip_logical_plan_aggregation() -> Result<()> {let plan = ctx.sql(query).await?.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));Ok(())@@ -380,7 +384,7 @@ async fn roundtrip_logical_plan_sort() -> Result<()> {let plan = ctx.sql(query).await?.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));Ok(())@@ -412,7 +416,7 @@ async fn roundtrip_logical_plan_dml() -> Result<()> {for query in queries {let plan = ctx.sql(query).await?.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(format!("{plan}"),format!("{logical_round_trip}"),@@ -442,7 +446,7 @@ async fn roundtrip_logical_plan_copy_to_sql_options() -> Result<()> {let codec = CsvLogicalExtensionCodec {};let bytes = logical_plan_to_bytes_with_extension_codec(&plan, &codec)?;let logical_round_trip =- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));Ok(())@@ -482,7 +486,7 @@ async fn roundtrip_logical_plan_copy_to_writer_options() -> Result<()> {let codec = ParquetLogicalExtensionCodec {};let bytes = logical_plan_to_bytes_with_extension_codec(&plan, &codec)?;let logical_round_trip =- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));match logical_round_trip {LogicalPlan::Copy(copy_to) => {@@ -514,7 +518,7 @@ async fn roundtrip_logical_plan_copy_to_arrow() -> Result<()> {let codec = ArrowLogicalExtensionCodec {};let bytes = logical_plan_to_bytes_with_extension_codec(&plan, &codec)?;let logical_round_trip =- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));match logical_round_trip {@@ -561,7 +565,7 @@ async fn roundtrip_logical_plan_copy_to_csv() -> Result<()> {let codec = CsvLogicalExtensionCodec {};let bytes = logical_plan_to_bytes_with_extension_codec(&plan, &codec)?;let logical_round_trip =- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));match logical_round_trip {@@ -628,7 +632,7 @@ async fn roundtrip_logical_plan_copy_to_json() -> Result<()> {let codec = JsonLogicalExtensionCodec {};let bytes = logical_plan_to_bytes_with_extension_codec(&plan, &codec)?;let logical_round_trip =- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));match logical_round_trip {@@ -700,7 +704,7 @@ async fn roundtrip_logical_plan_copy_to_parquet() -> Result<()> {let codec = ParquetLogicalExtensionCodec {};let bytes = logical_plan_to_bytes_with_extension_codec(&plan, &codec)?;let logical_round_trip =- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));match logical_round_trip {@@ -787,7 +791,7 @@ async fn roundtrip_logical_plan_distinct_on() -> Result<()> {let plan = ctx.sql(query).await?.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));Ok(())@@ -813,7 +817,7 @@ async fn roundtrip_single_count_distinct() -> Result<()> {let plan = ctx.sql(query).await?.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));Ok(())@@ -826,7 +830,7 @@ async fn roundtrip_logical_plan_with_extension() -> Result<()> {.await?;let plan = ctx.table("t1").await?.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));Ok(())}@@ -851,7 +855,7 @@ async fn roundtrip_logical_plan_unnest() -> Result<()> {let query = "SELECT unnest(b) FROM t1";let plan = ctx.sql(query).await?.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));Ok(())}@@ -1039,7 +1043,7 @@ async fn roundtrip_expr_api() -> Result<()> {// ensure expressions created with the expr api can be round trippedlet plan = table.select(expr_list)?.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));Ok(())}@@ -1059,13 +1063,13 @@ async fn roundtrip_logical_plan_with_view_scan() -> Result<()> {.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));// DROPlet plan = ctx.sql("DROP VIEW view_t1").await?.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));Ok(())@@ -1160,7 +1164,7 @@ impl LogicalExtensionCodec for TopKExtensionCodec {&self,buf: &[u8],inputs: &[LogicalPlan],- ctx: &SessionContext,+ ctx: &TaskContext,) -> Result<Extension> {if let Some((input, _)) = inputs.split_first() {let proto = proto::TopKPlanProto::decode(buf).map_err(|e| {@@ -1207,7 +1211,7 @@ impl LogicalExtensionCodec for TopKExtensionCodec {_buf: &[u8],_table_ref: &TableReference,_schema: SchemaRef,- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> Result<Arc<dyn TableProvider>> {internal_err!("unsupported plan type")}@@ -1230,7 +1234,7 @@ impl LogicalExtensionCodec for UDFExtensionCodec {&self,_buf: &[u8],_inputs: &[LogicalPlan],- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> Result<Extension> {not_impl_err!("No extension codec provided")}@@ -1244,7 +1248,7 @@ impl LogicalExtensionCodec for UDFExtensionCodec {_buf: &[u8],_table_ref: &TableReference,_schema: SchemaRef,- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> Result<Arc<dyn TableProvider>> {internal_err!("unsupported plan type")}@@ -2269,7 +2273,7 @@ fn roundtrip_scalar_udf() {&self,_buf: &[u8],_inputs: &[LogicalPlan],- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> Result<Extension> {not_impl_err!("LogicalExtensionCodec is not provided")}@@ -2283,7 +2287,7 @@ fn roundtrip_scalar_udf() {_buf: &[u8],_table_ref: &TableReference,_schema: SchemaRef,- _ctx: &SessionContext,+ _ctx: &TaskContext,) -> Result<Arc<dyn TableProvider>> {not_impl_err!("LogicalExtensionCodec is not provided")}@@ -2677,7 +2681,7 @@ async fn roundtrip_recursive_query() {let bytes = logical_plan_to_bytes(&plan).unwrap();let ctx = SessionContext::new();- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx).unwrap();+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx()).unwrap();assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));let dataframe = ctx.execute_logical_plan(logical_round_trip).await.unwrap();let output_round_trip = dataframe.collect().await.unwrap();@@ -2708,7 +2712,7 @@ async fn roundtrip_union_query() -> Result<()> {.await?;ctx.register_csv("t2", "tests/testdata/test.csv", CsvReadOptions::default()).await?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;// proto deserialization only supports 2-way union, hence this plan has nested unions// apply the flatten unions optimizer rule to be able to comparelet optimizer = Optimizer::with_rules(vec![Arc::new(EliminateNestedUnion::new())]);@@ -2745,7 +2749,7 @@ async fn roundtrip_custom_listing_tables_schema() -> Result<()> {.clone();let bytes = logical_plan_to_bytes(&plan)?;- let new_plan = logical_plan_from_bytes(&bytes, &ctx)?;+ let new_plan = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(plan, new_plan);Ok(())}@@ -2782,7 +2786,7 @@ async fn roundtrip_custom_listing_tables_schema_table_scan_projection() -> Resul.build()?;let bytes = logical_plan_to_bytes(&plan)?;- let new_plan = logical_plan_from_bytes(&bytes, &ctx)?;+ let new_plan = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(plan, new_plan);Ok(())@@ -2796,7 +2800,7 @@ async fn roundtrip_arrow_scan() -> Result<()> {.await?.into_optimized_plan()?;let bytes = logical_plan_to_bytes(&plan)?;- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));Ok(())}diff --git a/datafusion/proto/tests/cases/serialize.rs b/datafusion/proto/tests/cases/serialize.rsindex c9ef4377d4..3d69183668 100644--- a/datafusion/proto/tests/cases/serialize.rs+++ b/datafusion/proto/tests/cases/serialize.rs@@ -61,7 +61,7 @@ fn json_to_plan() {let input = r#"{"emptyRelation":{}}"#.to_string();let ctx = SessionContext::new();- let actual = logical_plan_from_json(&input, &ctx).unwrap();+ let actual = logical_plan_from_json(&input, &ctx.task_ctx()).unwrap();let result = matches!(actual, LogicalPlan::EmptyRelation(_));assert!(result, "Should parse empty relation");}diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.mdindex 0b9da1b5a8..8b03193e7f 100644--- a/docs/source/library-user-guide/upgrading.md+++ b/docs/source/library-user-guide/upgrading.md@@ -94,6 +94,28 @@ has changed from `&str` to `&Dialect`.crate under the `config` module that provides type safetyand better validation for SQL dialect selection+### Reorganization of `ListingTable` into `datafusion-catalog-listing` crate++There has been a long standing request to remove features such as `ListingTable`+from the `datafusion` crate to support faster build times. The structs+`ListingOptions`, `ListingTable`, and `ListingTableConfig` are now available+within the `datafusion-catalog-listing` crate. These are re-exported in+the `datafusion` crate, so this should be a minimal impact to existing users.++See [issue #14462] and [issue #17713] for more details.++[issue #14462]: https://github.com/apache/datafusion/issues/14462+[issue #17713]: https://github.com/apache/datafusion/issues/17713++### Reorganization of `ArrowSource` into `datafusion-datasource-arrow` crate++To support [issue #17713] the `ArrowSource` code has been removed from+the `datafusion` core crate into it's own crate, `datafusion-datasource-arrow`.+This follows the pattern for the AVRO, CSV, JSON, and Parquet data sources.+Users may need to update their paths to account for these changes.++See [issue #17713] for more details.+## DataFusion `50.0.0`### ListingTable automatically detects Hive Partitioned tables