Skip to content

Apache DataFusion SQL Query Engine

refactor: remove core crate from datafusion-proto (#18123)

Tim Saucerf199b0008613
diff --git a/Cargo.lock b/Cargo.lock
index 7e1d2b332a..13fe25c914 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2533,11 +2533,24 @@ dependencies = [
"arrow",
"chrono",
"datafusion",
+ "datafusion-catalog",
+ "datafusion-catalog-listing",
"datafusion-common",
+ "datafusion-datasource",
+ "datafusion-datasource-arrow",
+ "datafusion-datasource-avro",
+ "datafusion-datasource-csv",
+ "datafusion-datasource-json",
+ "datafusion-datasource-parquet",
+ "datafusion-execution",
"datafusion-expr",
"datafusion-functions",
"datafusion-functions-aggregate",
+ "datafusion-functions-table",
"datafusion-functions-window-common",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
"datafusion-proto-common",
"doc-comment",
"object_store",
diff --git a/benchmarks/src/imdb/run.rs b/benchmarks/src/imdb/run.rs
index 3d58d5f54d..11bd424ba6 100644
--- a/benchmarks/src/imdb/run.rs
+++ b/benchmarks/src/imdb/run.rs
@@ -534,7 +534,7 @@ mod tests {
let plan = ctx.sql(&query).await?;
let plan = plan.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let plan2 = logical_plan_from_bytes(&bytes, &ctx)?;
+ let plan2 = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
let plan_formatted = format!("{}", plan.display_indent());
let plan2_formatted = format!("{}", plan2.display_indent());
assert_eq!(plan_formatted, plan2_formatted);
diff --git a/benchmarks/src/tpch/run.rs b/benchmarks/src/tpch/run.rs
index b93bdf254a..2b66de641b 100644
--- a/benchmarks/src/tpch/run.rs
+++ b/benchmarks/src/tpch/run.rs
@@ -387,7 +387,7 @@ mod tests {
let plan = ctx.sql(&query).await?;
let plan = plan.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let plan2 = logical_plan_from_bytes(&bytes, &ctx)?;
+ let plan2 = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
let plan_formatted = format!("{}", plan.display_indent());
let plan2_formatted = format!("{}", plan2.display_indent());
assert_eq!(plan_formatted, plan2_formatted);
diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml
index c1d894a6c0..920e277b8c 100644
--- a/datafusion/proto/Cargo.toml
+++ b/datafusion/proto/Cargo.toml
@@ -40,15 +40,31 @@ name = "datafusion_proto"
[features]
default = ["parquet"]
json = ["pbjson", "serde", "serde_json", "datafusion-proto-common/json"]
-parquet = ["datafusion/parquet", "datafusion-common/parquet"]
-avro = ["datafusion/avro", "datafusion-common/avro"]
+parquet = ["datafusion-datasource-parquet", "datafusion-common/parquet", "datafusion/parquet"]
+avro = ["datafusion-datasource-avro", "datafusion-common/avro"]
+
+# Note to developers: do *not* add `datafusion` as a dependency in
+# this crate. See https://github.com/apache/datafusion/issues/17713
+# for additional information.
[dependencies]
arrow = { workspace = true }
chrono = { workspace = true }
-datafusion = { workspace = true, default-features = false }
+datafusion-catalog = { workspace = true }
+datafusion-catalog-listing = { workspace = true }
datafusion-common = { workspace = true }
+datafusion-datasource = { workspace = true }
+datafusion-datasource-arrow = { workspace = true }
+datafusion-datasource-avro = { workspace = true, optional = true }
+datafusion-datasource-csv = { workspace = true }
+datafusion-datasource-json = { workspace = true }
+datafusion-datasource-parquet = { workspace = true, optional = true }
+datafusion-execution = { workspace = true }
datafusion-expr = { workspace = true }
+datafusion-functions-table = { workspace = true }
+datafusion-physical-expr = { workspace = true }
+datafusion-physical-expr-common = { workspace = true }
+datafusion-physical-plan = { workspace = true }
datafusion-proto-common = { workspace = true }
object_store = { workspace = true }
pbjson = { workspace = true, optional = true }
diff --git a/datafusion/proto/src/bytes/mod.rs b/datafusion/proto/src/bytes/mod.rs
index 5b07e59e80..12d9938373 100644
--- a/datafusion/proto/src/bytes/mod.rs
+++ b/datafusion/proto/src/bytes/mod.rs
@@ -24,8 +24,8 @@ use crate::physical_plan::{
AsExecutionPlan, DefaultPhysicalExtensionCodec, PhysicalExtensionCodec,
};
use crate::protobuf;
-use datafusion::execution::TaskContext;
use datafusion_common::{plan_datafusion_err, Result};
+use datafusion_execution::TaskContext;
use datafusion_expr::{
create_udaf, create_udf, create_udwf, AggregateUDF, Expr, LogicalPlan, Volatility,
WindowUDF,
@@ -37,10 +37,9 @@ use prost::{
use std::sync::Arc;
// Reexport Bytes which appears in the API
-use datafusion::execution::registry::FunctionRegistry;
-use datafusion::physical_plan::ExecutionPlan;
-use datafusion::prelude::SessionContext;
+use datafusion_execution::registry::FunctionRegistry;
use datafusion_expr::planner::ExprPlanner;
+use datafusion_physical_plan::ExecutionPlan;
mod registry;
@@ -240,16 +239,13 @@ pub fn logical_plan_to_json_with_extension_codec(
/// Deserialize a LogicalPlan from JSON
#[cfg(feature = "json")]
-pub fn logical_plan_from_json(json: &str, ctx: &SessionContext) -> Result<LogicalPlan> {
+pub fn logical_plan_from_json(json: &str, ctx: &TaskContext) -> Result<LogicalPlan> {
let extension_codec = DefaultLogicalExtensionCodec {};
logical_plan_from_json_with_extension_codec(json, ctx, &extension_codec)
}
/// Deserialize a LogicalPlan from bytes
-pub fn logical_plan_from_bytes(
- bytes: &[u8],
- ctx: &SessionContext,
-) -> Result<LogicalPlan> {
+pub fn logical_plan_from_bytes(bytes: &[u8], ctx: &TaskContext) -> Result<LogicalPlan> {
let extension_codec = DefaultLogicalExtensionCodec {};
logical_plan_from_bytes_with_extension_codec(bytes, ctx, &extension_codec)
}
@@ -257,7 +253,7 @@ pub fn logical_plan_from_bytes(
/// Deserialize a LogicalPlan from bytes
pub fn logical_plan_from_bytes_with_extension_codec(
bytes: &[u8],
- ctx: &SessionContext,
+ ctx: &TaskContext,
extension_codec: &dyn LogicalExtensionCodec,
) -> Result<LogicalPlan> {
let protobuf = protobuf::LogicalPlanNode::decode(bytes)
@@ -269,7 +265,7 @@ pub fn logical_plan_from_bytes_with_extension_codec(
#[cfg(feature = "json")]
pub fn logical_plan_from_json_with_extension_codec(
json: &str,
- ctx: &SessionContext,
+ ctx: &TaskContext,
extension_codec: &dyn LogicalExtensionCodec,
) -> Result<LogicalPlan> {
let back: protobuf::LogicalPlanNode = serde_json::from_str(json)
@@ -312,12 +308,12 @@ pub fn physical_plan_to_bytes_with_extension_codec(
#[cfg(feature = "json")]
pub fn physical_plan_from_json(
json: &str,
- ctx: &SessionContext,
+ ctx: &TaskContext,
) -> Result<Arc<dyn ExecutionPlan>> {
let back: protobuf::PhysicalPlanNode = serde_json::from_str(json)
.map_err(|e| plan_datafusion_err!("Error serializing plan: {e}"))?;
let extension_codec = DefaultPhysicalExtensionCodec {};
- back.try_into_physical_plan(&ctx.task_ctx(), &extension_codec)
+ back.try_into_physical_plan(&ctx, &extension_codec)
}
/// Deserialize a PhysicalPlan from bytes
diff --git a/datafusion/proto/src/bytes/registry.rs b/datafusion/proto/src/bytes/registry.rs
index 5d46d41f79..087e073db2 100644
--- a/datafusion/proto/src/bytes/registry.rs
+++ b/datafusion/proto/src/bytes/registry.rs
@@ -17,9 +17,9 @@
use std::{collections::HashSet, sync::Arc};
-use datafusion::execution::registry::FunctionRegistry;
use datafusion_common::plan_err;
use datafusion_common::Result;
+use datafusion_execution::registry::FunctionRegistry;
use datafusion_expr::planner::ExprPlanner;
use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
diff --git a/datafusion/proto/src/lib.rs b/datafusion/proto/src/lib.rs
index bb7b992f14..b1590b9ad2 100644
--- a/datafusion/proto/src/lib.rs
+++ b/datafusion/proto/src/lib.rs
@@ -34,8 +34,8 @@
//!
//! [`LogicalPlan`]: datafusion_expr::LogicalPlan
//! [`Expr`]: datafusion_expr::Expr
-//! [`ExecutionPlan`]: datafusion::physical_plan::ExecutionPlan
-//! [`PhysicalExpr`]: datafusion::physical_expr::PhysicalExpr
+//! [`ExecutionPlan`]: datafusion_physical_plan::ExecutionPlan
+//! [`PhysicalExpr`]: datafusion_physical_expr::PhysicalExpr
//!
//! Internally, this crate is implemented by converting the plans to [protocol
//! buffers] using [prost].
@@ -93,7 +93,7 @@
//! let bytes = logical_plan_to_bytes(&plan)?;
//!
//! // Decode bytes from somewhere (over network, etc.) back to LogicalPlan
-//! let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+//! let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
//! assert_eq!(format!("{:?}", plan), format!("{:?}", logical_round_trip));
//! # Ok(())
//! # }
diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rs
index 0e76e19ecb..d32bfb22ff 100644
--- a/datafusion/proto/src/logical_plan/file_formats.rs
+++ b/datafusion/proto/src/logical_plan/file_formats.rs
@@ -17,22 +17,19 @@
use std::sync::Arc;
-use datafusion::{
- config::{CsvOptions, JsonOptions},
- datasource::file_format::{
- arrow::ArrowFormatFactory, csv::CsvFormatFactory, json::JsonFormatFactory,
- FileFormatFactory,
- },
- prelude::SessionContext,
-};
+use crate::protobuf::{CsvOptions as CsvOptionsProto, JsonOptions as JsonOptionsProto};
+use datafusion_common::config::{CsvOptions, JsonOptions};
use datafusion_common::{
exec_datafusion_err, exec_err, not_impl_err, parsers::CompressionTypeVariant,
TableReference,
};
+use datafusion_datasource::file_format::FileFormatFactory;
+use datafusion_datasource_arrow::file_format::ArrowFormatFactory;
+use datafusion_datasource_csv::file_format::CsvFormatFactory;
+use datafusion_datasource_json::file_format::JsonFormatFactory;
+use datafusion_execution::TaskContext;
use prost::Message;
-use crate::protobuf::{CsvOptions as CsvOptionsProto, JsonOptions as JsonOptionsProto};
-
use super::LogicalExtensionCodec;
#[derive(Debug)]
@@ -165,7 +162,7 @@ impl LogicalExtensionCodec for CsvLogicalExtensionCodec {
&self,
_buf: &[u8],
_inputs: &[datafusion_expr::LogicalPlan],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> datafusion_common::Result<datafusion_expr::Extension> {
not_impl_err!("Method not implemented")
}
@@ -183,15 +180,15 @@ impl LogicalExtensionCodec for CsvLogicalExtensionCodec {
_buf: &[u8],
_table_ref: &TableReference,
_schema: arrow::datatypes::SchemaRef,
- _ctx: &SessionContext,
- ) -> datafusion_common::Result<Arc<dyn datafusion::datasource::TableProvider>> {
+ _ctx: &TaskContext,
+ ) -> datafusion_common::Result<Arc<dyn datafusion_catalog::TableProvider>> {
not_impl_err!("Method not implemented")
}
fn try_encode_table_provider(
&self,
_table_ref: &TableReference,
- _node: Arc<dyn datafusion::datasource::TableProvider>,
+ _node: Arc<dyn datafusion_catalog::TableProvider>,
_buf: &mut Vec<u8>,
) -> datafusion_common::Result<()> {
not_impl_err!("Method not implemented")
@@ -200,7 +197,7 @@ impl LogicalExtensionCodec for CsvLogicalExtensionCodec {
fn try_decode_file_format(
&self,
buf: &[u8],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> datafusion_common::Result<Arc<dyn FileFormatFactory>> {
let proto = CsvOptionsProto::decode(buf).map_err(|e| {
exec_datafusion_err!("Failed to decode CsvOptionsProto: {e:?}")
@@ -272,7 +269,7 @@ impl LogicalExtensionCodec for JsonLogicalExtensionCodec {
&self,
_buf: &[u8],
_inputs: &[datafusion_expr::LogicalPlan],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> datafusion_common::Result<datafusion_expr::Extension> {
not_impl_err!("Method not implemented")
}
@@ -290,15 +287,15 @@ impl LogicalExtensionCodec for JsonLogicalExtensionCodec {
_buf: &[u8],
_table_ref: &TableReference,
_schema: arrow::datatypes::SchemaRef,
- _ctx: &SessionContext,
- ) -> datafusion_common::Result<Arc<dyn datafusion::datasource::TableProvider>> {
+ _ctx: &TaskContext,
+ ) -> datafusion_common::Result<Arc<dyn datafusion_catalog::TableProvider>> {
not_impl_err!("Method not implemented")
}
fn try_encode_table_provider(
&self,
_table_ref: &TableReference,
- _node: Arc<dyn datafusion::datasource::TableProvider>,
+ _node: Arc<dyn datafusion_catalog::TableProvider>,
_buf: &mut Vec<u8>,
) -> datafusion_common::Result<()> {
not_impl_err!("Method not implemented")
@@ -307,7 +304,7 @@ impl LogicalExtensionCodec for JsonLogicalExtensionCodec {
fn try_decode_file_format(
&self,
buf: &[u8],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> datafusion_common::Result<Arc<dyn FileFormatFactory>> {
let proto = JsonOptionsProto::decode(buf).map_err(|e| {
exec_datafusion_err!("Failed to decode JsonOptionsProto: {e:?}")
@@ -353,11 +350,10 @@ mod parquet {
ParquetOptions as ParquetOptionsProto,
TableParquetOptions as TableParquetOptionsProto,
};
-
- use datafusion::{
- config::{ParquetColumnOptions, ParquetOptions, TableParquetOptions},
- datasource::file_format::parquet::ParquetFormatFactory,
+ use datafusion_common::config::{
+ ParquetColumnOptions, ParquetOptions, TableParquetOptions,
};
+ use datafusion_datasource_parquet::file_format::ParquetFormatFactory;
impl TableParquetOptionsProto {
fn from_factory(factory: &ParquetFormatFactory) -> Self {
@@ -595,7 +591,7 @@ mod parquet {
&self,
_buf: &[u8],
_inputs: &[datafusion_expr::LogicalPlan],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> datafusion_common::Result<datafusion_expr::Extension> {
not_impl_err!("Method not implemented")
}
@@ -613,8 +609,8 @@ mod parquet {
_buf: &[u8],
_table_ref: &TableReference,
_schema: arrow::datatypes::SchemaRef,
- _ctx: &SessionContext,
- ) -> datafusion_common::Result<Arc<dyn datafusion::datasource::TableProvider>>
+ _ctx: &TaskContext,
+ ) -> datafusion_common::Result<Arc<dyn datafusion_catalog::TableProvider>>
{
not_impl_err!("Method not implemented")
}
@@ -622,7 +618,7 @@ mod parquet {
fn try_encode_table_provider(
&self,
_table_ref: &TableReference,
- _node: Arc<dyn datafusion::datasource::TableProvider>,
+ _node: Arc<dyn datafusion_catalog::TableProvider>,
_buf: &mut Vec<u8>,
) -> datafusion_common::Result<()> {
not_impl_err!("Method not implemented")
@@ -631,14 +627,14 @@ mod parquet {
fn try_decode_file_format(
&self,
buf: &[u8],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> datafusion_common::Result<Arc<dyn FileFormatFactory>> {
let proto = TableParquetOptionsProto::decode(buf).map_err(|e| {
exec_datafusion_err!("Failed to decode TableParquetOptionsProto: {e:?}")
})?;
let options: TableParquetOptions = (&proto).into();
Ok(Arc::new(
- datafusion::datasource::file_format::parquet::ParquetFormatFactory {
+ datafusion_datasource_parquet::file_format::ParquetFormatFactory {
options: Some(options),
},
))
@@ -649,7 +645,7 @@ mod parquet {
buf: &mut Vec<u8>,
node: Arc<dyn FileFormatFactory>,
) -> datafusion_common::Result<()> {
- use datafusion::datasource::file_format::parquet::ParquetFormatFactory;
+ use datafusion_datasource_parquet::file_format::ParquetFormatFactory;
let options = if let Some(parquet_factory) =
node.as_any().downcast_ref::<ParquetFormatFactory>()
@@ -683,7 +679,7 @@ impl LogicalExtensionCodec for ArrowLogicalExtensionCodec {
&self,
_buf: &[u8],
_inputs: &[datafusion_expr::LogicalPlan],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> datafusion_common::Result<datafusion_expr::Extension> {
not_impl_err!("Method not implemented")
}
@@ -701,15 +697,15 @@ impl LogicalExtensionCodec for ArrowLogicalExtensionCodec {
_buf: &[u8],
_table_ref: &TableReference,
_schema: arrow::datatypes::SchemaRef,
- _ctx: &SessionContext,
- ) -> datafusion_common::Result<Arc<dyn datafusion::datasource::TableProvider>> {
+ _ctx: &TaskContext,
+ ) -> datafusion_common::Result<Arc<dyn datafusion_catalog::TableProvider>> {
not_impl_err!("Method not implemented")
}
fn try_encode_table_provider(
&self,
_table_ref: &TableReference,
- _node: Arc<dyn datafusion::datasource::TableProvider>,
+ _node: Arc<dyn datafusion_catalog::TableProvider>,
_buf: &mut Vec<u8>,
) -> datafusion_common::Result<()> {
not_impl_err!("Method not implemented")
@@ -718,7 +714,7 @@ impl LogicalExtensionCodec for ArrowLogicalExtensionCodec {
fn try_decode_file_format(
&self,
__buf: &[u8],
- __ctx: &SessionContext,
+ __ctx: &TaskContext,
) -> datafusion_common::Result<Arc<dyn FileFormatFactory>> {
Ok(Arc::new(ArrowFormatFactory::new()))
}
@@ -741,7 +737,7 @@ impl LogicalExtensionCodec for AvroLogicalExtensionCodec {
&self,
_buf: &[u8],
_inputs: &[datafusion_expr::LogicalPlan],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> datafusion_common::Result<datafusion_expr::Extension> {
not_impl_err!("Method not implemented")
}
@@ -759,15 +755,15 @@ impl LogicalExtensionCodec for AvroLogicalExtensionCodec {
_buf: &[u8],
_table_ref: &TableReference,
_schema: arrow::datatypes::SchemaRef,
- _cts: &SessionContext,
- ) -> datafusion_common::Result<Arc<dyn datafusion::datasource::TableProvider>> {
+ _cts: &TaskContext,
+ ) -> datafusion_common::Result<Arc<dyn datafusion_catalog::TableProvider>> {
not_impl_err!("Method not implemented")
}
fn try_encode_table_provider(
&self,
_table_ref: &TableReference,
- _node: Arc<dyn datafusion::datasource::TableProvider>,
+ _node: Arc<dyn datafusion_catalog::TableProvider>,
_buf: &mut Vec<u8>,
) -> datafusion_common::Result<()> {
not_impl_err!("Method not implemented")
@@ -776,7 +772,7 @@ impl LogicalExtensionCodec for AvroLogicalExtensionCodec {
fn try_decode_file_format(
&self,
__buf: &[u8],
- __ctx: &SessionContext,
+ __ctx: &TaskContext,
) -> datafusion_common::Result<Arc<dyn FileFormatFactory>> {
Ok(Arc::new(ArrowFormatFactory::new()))
}
diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs
index ec6415adc4..4296867049 100644
--- a/datafusion/proto/src/logical_plan/from_proto.rs
+++ b/datafusion/proto/src/logical_plan/from_proto.rs
@@ -17,11 +17,11 @@
use std::sync::Arc;
-use datafusion::execution::registry::FunctionRegistry;
use datafusion_common::{
exec_datafusion_err, internal_err, plan_datafusion_err, NullEquality,
RecursionUnnestOption, Result, ScalarValue, TableReference, UnnestOptions,
};
+use datafusion_execution::registry::FunctionRegistry;
use datafusion_expr::dml::InsertOp;
use datafusion_expr::expr::{Alias, NullTreatment, Placeholder, Sort};
use datafusion_expr::expr::{Unnest, WildcardOptions};
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
index fd9e07914b..ad56185166 100644
--- a/datafusion/proto/src/logical_plan/mod.rs
+++ b/datafusion/proto/src/logical_plan/mod.rs
@@ -34,32 +34,23 @@ use crate::{
use crate::protobuf::{proto_error, ToProtoError};
use arrow::datatypes::{DataType, Schema, SchemaBuilder, SchemaRef};
-use datafusion::datasource::cte_worktable::CteWorkTable;
-use datafusion::datasource::file_format::arrow::ArrowFormat;
-#[cfg(feature = "avro")]
-use datafusion::datasource::file_format::avro::AvroFormat;
-#[cfg(feature = "parquet")]
-use datafusion::datasource::file_format::parquet::ParquetFormat;
-use datafusion::datasource::file_format::{
- file_type_to_format, format_as_file_type, FileFormatFactory,
-};
-use datafusion::{
- datasource::{
- file_format::{
- csv::CsvFormat, json::JsonFormat as OtherNdJsonFormat, FileFormat,
- },
- listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl},
- view::ViewTable,
- TableProvider,
- },
- datasource::{provider_as_source, source_as_provider},
- prelude::SessionContext,
-};
+use datafusion_catalog::cte_worktable::CteWorkTable;
use datafusion_common::file_options::file_type::FileType;
use datafusion_common::{
context, internal_datafusion_err, internal_err, not_impl_err, plan_err, Result,
TableReference, ToDFSchema,
};
+use datafusion_datasource::file_format::FileFormat;
+use datafusion_datasource::file_format::{
+ file_type_to_format, format_as_file_type, FileFormatFactory,
+};
+use datafusion_datasource_arrow::file_format::ArrowFormat;
+#[cfg(feature = "avro")]
+use datafusion_datasource_avro::file_format::AvroFormat;
+use datafusion_datasource_csv::file_format::CsvFormat;
+use datafusion_datasource_json::file_format::JsonFormat as OtherNdJsonFormat;
+#[cfg(feature = "parquet")]
+use datafusion_datasource_parquet::file_format::ParquetFormat;
use datafusion_expr::{
dml,
logical_plan::{
@@ -77,6 +68,12 @@ use datafusion_expr::{
use self::to_proto::{serialize_expr, serialize_exprs};
use crate::logical_plan::to_proto::serialize_sorts;
+use datafusion_catalog::default_table_source::{provider_as_source, source_as_provider};
+use datafusion_catalog::view::ViewTable;
+use datafusion_catalog::TableProvider;
+use datafusion_catalog_listing::{ListingOptions, ListingTable, ListingTableConfig};
+use datafusion_datasource::ListingTableUrl;
+use datafusion_execution::TaskContext;
use prost::bytes::BufMut;
use prost::Message;
@@ -96,7 +93,7 @@ pub trait AsLogicalPlan: Debug + Send + Sync + Clone {
fn try_into_logical_plan(
&self,
- ctx: &SessionContext,
+ ctx: &TaskContext,
extension_codec: &dyn LogicalExtensionCodec,
) -> Result<LogicalPlan>;
@@ -113,7 +110,7 @@ pub trait LogicalExtensionCodec: Debug + Send + Sync {
&self,
buf: &[u8],
inputs: &[LogicalPlan],
- ctx: &SessionContext,
+ ctx: &TaskContext,
) -> Result<Extension>;
fn try_encode(&self, node: &Extension, buf: &mut Vec<u8>) -> Result<()>;
@@ -123,7 +120,7 @@ pub trait LogicalExtensionCodec: Debug + Send + Sync {
buf: &[u8],
table_ref: &TableReference,
schema: SchemaRef,
- ctx: &SessionContext,
+ ctx: &TaskContext,
) -> Result<Arc<dyn TableProvider>>;
fn try_encode_table_provider(
@@ -136,7 +133,7 @@ pub trait LogicalExtensionCodec: Debug + Send + Sync {
fn try_decode_file_format(
&self,
_buf: &[u8],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> Result<Arc<dyn FileFormatFactory>> {
not_impl_err!("LogicalExtensionCodec is not provided for file format")
}
@@ -184,7 +181,7 @@ impl LogicalExtensionCodec for DefaultLogicalExtensionCodec {
&self,
_buf: &[u8],
_inputs: &[LogicalPlan],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> Result<Extension> {
not_impl_err!("LogicalExtensionCodec is not provided")
}
@@ -198,7 +195,7 @@ impl LogicalExtensionCodec for DefaultLogicalExtensionCodec {
_buf: &[u8],
_table_ref: &TableReference,
_schema: SchemaRef,
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> Result<Arc<dyn TableProvider>> {
not_impl_err!("LogicalExtensionCodec is not provided")
}
@@ -242,7 +239,7 @@ fn from_table_reference(
/// serialized by [from_table_source]
fn to_table_source(
node: &Option<Box<LogicalPlanNode>>,
- ctx: &SessionContext,
+ ctx: &TaskContext,
extension_codec: &dyn LogicalExtensionCodec,
) -> Result<Arc<dyn TableSource>> {
if let Some(node) = node {
@@ -296,7 +293,7 @@ impl AsLogicalPlan for LogicalPlanNode {
fn try_into_logical_plan(
&self,
- ctx: &SessionContext,
+ ctx: &TaskContext,
extension_codec: &dyn LogicalExtensionCodec,
) -> Result<LogicalPlan> {
let plan = self.logical_plan_type.as_ref().ok_or_else(|| {
@@ -475,10 +472,7 @@ impl AsLogicalPlan for LogicalPlanNode {
.with_schema(Arc::new(schema));
let provider = ListingTable::try_new(config)?.with_cache(
- ctx.state()
- .runtime_env()
- .cache_manager
- .get_file_statistic_cache(),
+ ctx.runtime_env().cache_manager.get_file_statistic_cache(),
);
let table_name =
@@ -547,7 +541,7 @@ impl AsLogicalPlan for LogicalPlanNode {
.build()
}
LogicalPlanType::Repartition(repartition) => {
- use datafusion::logical_expr::Partitioning;
+ use datafusion_expr::Partitioning;
let input: LogicalPlan =
into_logical_plan!(repartition.input, ctx, extension_codec)?;
use protobuf::repartition_node::PartitionMethod;
@@ -595,11 +589,6 @@ impl AsLogicalPlan for LogicalPlanNode {
None
};
- let file_type = create_extern_table.file_type.as_str();
- if ctx.table_factory(file_type).is_none() {
- internal_err!("No TableProviderFactory for file type: {file_type}")?
- }
-
let mut order_exprs = vec![];
for expr in &create_extern_table.order_exprs {
order_exprs.push(from_proto::parse_sorts(
@@ -961,14 +950,14 @@ impl AsLogicalPlan for LogicalPlanNode {
)?
.build()
}
- LogicalPlanType::Dml(dml_node) => Ok(LogicalPlan::Dml(
- datafusion::logical_expr::DmlStatement::new(
+ LogicalPlanType::Dml(dml_node) => {
+ Ok(LogicalPlan::Dml(datafusion_expr::DmlStatement::new(
from_table_reference(dml_node.table_name.as_ref(), "DML ")?,
to_table_source(&dml_node.target, ctx, extension_codec)?,
dml_node.dml_type().into(),
Arc::new(into_logical_plan!(dml_node.input, ctx, extension_codec)?),
- ),
- )),
+ )))
+ }
}
}
@@ -1417,7 +1406,7 @@ impl AsLogicalPlan for LogicalPlanNode {
input,
partitioning_scheme,
}) => {
- use datafusion::logical_expr::Partitioning;
+ use datafusion_expr::Partitioning;
let input: LogicalPlanNode = LogicalPlanNode::try_from_logical_plan(
input.as_ref(),
extension_codec,
diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rs
index e2ee1be7d7..7c4b9e55b8 100644
--- a/datafusion/proto/src/physical_plan/from_proto.rs
+++ b/datafusion/proto/src/physical_plan/from_proto.rs
@@ -28,26 +28,27 @@ use datafusion_expr::dml::InsertOp;
use object_store::path::Path;
use object_store::ObjectMeta;
-use datafusion::arrow::datatypes::Schema;
-use datafusion::datasource::file_format::csv::CsvSink;
-use datafusion::datasource::file_format::json::JsonSink;
+use arrow::datatypes::Schema;
+use datafusion_common::{internal_datafusion_err, not_impl_err, DataFusionError, Result};
+use datafusion_datasource::file::FileSource;
+use datafusion_datasource::file_groups::FileGroup;
+use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
+use datafusion_datasource::file_sink_config::FileSinkConfig;
+use datafusion_datasource::{FileRange, ListingTableUrl, PartitionedFile};
+use datafusion_datasource_csv::file_format::CsvSink;
+use datafusion_datasource_json::file_format::JsonSink;
#[cfg(feature = "parquet")]
-use datafusion::datasource::file_format::parquet::ParquetSink;
-use datafusion::datasource::listing::{FileRange, ListingTableUrl, PartitionedFile};
-use datafusion::datasource::object_store::ObjectStoreUrl;
-use datafusion::datasource::physical_plan::{
- FileGroup, FileScanConfig, FileScanConfigBuilder, FileSinkConfig, FileSource,
-};
-use datafusion::execution::{FunctionRegistry, TaskContext};
-use datafusion::logical_expr::WindowFunctionDefinition;
-use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr, ScalarFunctionExpr};
-use datafusion::physical_plan::expressions::{
+use datafusion_datasource_parquet::file_format::ParquetSink;
+use datafusion_execution::object_store::ObjectStoreUrl;
+use datafusion_execution::{FunctionRegistry, TaskContext};
+use datafusion_expr::WindowFunctionDefinition;
+use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr, ScalarFunctionExpr};
+use datafusion_physical_plan::expressions::{
in_list, BinaryExpr, CaseExpr, CastExpr, Column, IsNotNullExpr, IsNullExpr, LikeExpr,
Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,
};
-use datafusion::physical_plan::windows::{create_window_expr, schema_add_window_field};
-use datafusion::physical_plan::{Partitioning, PhysicalExpr, WindowExpr};
-use datafusion_common::{internal_datafusion_err, not_impl_err, DataFusionError, Result};
+use datafusion_physical_plan::windows::{create_window_expr, schema_add_window_field};
+use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};
use datafusion_proto_common::common::proto_error;
use crate::convert_required;
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index 818109be68..cd9bd67285 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -23,8 +23,7 @@ use self::to_proto::{serialize_partitioning, serialize_physical_expr};
use crate::common::{byte_to_string, str_to_byte};
use crate::physical_plan::from_proto::{
parse_physical_expr, parse_physical_sort_expr, parse_physical_sort_exprs,
- parse_physical_window_expr, parse_protobuf_file_scan_config,
- parse_protobuf_file_scan_schema, parse_record_batches,
+ parse_physical_window_expr, parse_protobuf_file_scan_config, parse_record_batches,
};
use crate::physical_plan::to_proto::{
serialize_file_scan_config, serialize_maybe_filter, serialize_physical_aggr_expr,
@@ -40,65 +39,64 @@ use crate::protobuf::{
};
use crate::{convert_required, into_required};
-use datafusion::arrow::compute::SortOptions;
-use datafusion::arrow::datatypes::{IntervalMonthDayNanoType, Schema, SchemaRef};
-use datafusion::catalog::memory::MemorySourceConfig;
-use datafusion::datasource::file_format::csv::CsvSink;
-use datafusion::datasource::file_format::file_compression_type::FileCompressionType;
-use datafusion::datasource::file_format::json::JsonSink;
+use arrow::compute::SortOptions;
+use arrow::datatypes::{IntervalMonthDayNanoType, SchemaRef};
+use datafusion_catalog::memory::MemorySourceConfig;
+use datafusion_common::{
+ internal_datafusion_err, internal_err, not_impl_err, DataFusionError, Result,
+};
#[cfg(feature = "parquet")]
-use datafusion::datasource::file_format::parquet::ParquetSink;
+use datafusion_datasource::file::FileSource;
+use datafusion_datasource::file_compression_type::FileCompressionType;
+use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
+use datafusion_datasource::sink::DataSinkExec;
+use datafusion_datasource::source::{DataSource, DataSourceExec};
#[cfg(feature = "avro")]
-use datafusion::datasource::physical_plan::AvroSource;
+use datafusion_datasource_avro::source::AvroSource;
+use datafusion_datasource_csv::file_format::CsvSink;
+use datafusion_datasource_csv::source::CsvSource;
+use datafusion_datasource_json::file_format::JsonSink;
+use datafusion_datasource_json::source::JsonSource;
#[cfg(feature = "parquet")]
-use datafusion::datasource::physical_plan::ParquetSource;
-use datafusion::datasource::physical_plan::{
- CsvSource, FileScanConfig, FileScanConfigBuilder, FileSource, JsonSource,
-};
-use datafusion::datasource::sink::DataSinkExec;
-use datafusion::datasource::source::{DataSource, DataSourceExec};
-use datafusion::execution::{FunctionRegistry, TaskContext};
-use datafusion::functions_table::generate_series::{
+use datafusion_datasource_parquet::file_format::ParquetSink;
+#[cfg(feature = "parquet")]
+use datafusion_datasource_parquet::source::ParquetSource;
+use datafusion_execution::{FunctionRegistry, TaskContext};
+use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
+use datafusion_functions_table::generate_series::{
Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue,
};
-use datafusion::physical_expr::aggregate::AggregateExprBuilder;
-use datafusion::physical_expr::aggregate::AggregateFunctionExpr;
-use datafusion::physical_expr::{LexOrdering, LexRequirement, PhysicalExprRef};
-use datafusion::physical_plan::aggregates::AggregateMode;
-use datafusion::physical_plan::aggregates::{AggregateExec, PhysicalGroupBy};
-use datafusion::physical_plan::analyze::AnalyzeExec;
-use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
-use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
-use datafusion::physical_plan::coop::CooperativeExec;
-use datafusion::physical_plan::empty::EmptyExec;
-use datafusion::physical_plan::explain::ExplainExec;
-use datafusion::physical_plan::expressions::PhysicalSortExpr;
-use datafusion::physical_plan::filter::FilterExec;
-use datafusion::physical_plan::joins::utils::{ColumnIndex, JoinFilter};
-use datafusion::physical_plan::joins::{
+use datafusion_physical_expr::aggregate::AggregateExprBuilder;
+use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
+use datafusion_physical_expr::{LexOrdering, LexRequirement, PhysicalExprRef};
+use datafusion_physical_plan::aggregates::AggregateMode;
+use datafusion_physical_plan::aggregates::{AggregateExec, PhysicalGroupBy};
+use datafusion_physical_plan::analyze::AnalyzeExec;
+use datafusion_physical_plan::coalesce_batches::CoalesceBatchesExec;
+use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;
+use datafusion_physical_plan::coop::CooperativeExec;
+use datafusion_physical_plan::empty::EmptyExec;
+use datafusion_physical_plan::explain::ExplainExec;
+use datafusion_physical_plan::expressions::PhysicalSortExpr;
+use datafusion_physical_plan::filter::FilterExec;
+use datafusion_physical_plan::joins::utils::{ColumnIndex, JoinFilter};
+use datafusion_physical_plan::joins::{
CrossJoinExec, NestedLoopJoinExec, SortMergeJoinExec, StreamJoinPartitionMode,
SymmetricHashJoinExec,
};
-use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode};
-use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
-use datafusion::physical_plan::memory::LazyMemoryExec;
-use datafusion::physical_plan::metrics::MetricType;
-use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;
-use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};
-use datafusion::physical_plan::repartition::RepartitionExec;
-use datafusion::physical_plan::sorts::sort::SortExec;
-use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
-use datafusion::physical_plan::union::{InterleaveExec, UnionExec};
-use datafusion::physical_plan::unnest::{ListUnnest, UnnestExec};
-use datafusion::physical_plan::windows::{BoundedWindowAggExec, WindowAggExec};
-use datafusion::physical_plan::{
- ExecutionPlan, InputOrderMode, PhysicalExpr, WindowExpr,
-};
-use datafusion_common::config::TableParquetOptions;
-use datafusion_common::{
- internal_datafusion_err, internal_err, not_impl_err, DataFusionError, Result,
-};
-use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
+use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};
+use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
+use datafusion_physical_plan::memory::LazyMemoryExec;
+use datafusion_physical_plan::metrics::MetricType;
+use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
+use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};
+use datafusion_physical_plan::repartition::RepartitionExec;
+use datafusion_physical_plan::sorts::sort::SortExec;
+use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
+use datafusion_physical_plan::union::{InterleaveExec, UnionExec};
+use datafusion_physical_plan::unnest::{ListUnnest, UnnestExec};
+use datafusion_physical_plan::windows::{BoundedWindowAggExec, WindowAggExec};
+use datafusion_physical_plan::{ExecutionPlan, InputOrderMode, PhysicalExpr, WindowExpr};
use prost::bytes::BufMut;
use prost::Message;
@@ -662,8 +660,9 @@ impl protobuf::PhysicalPlanNode {
) -> Result<Arc<dyn ExecutionPlan>> {
#[cfg(feature = "parquet")]
{
- let schema =
- parse_protobuf_file_scan_schema(scan.base_conf.as_ref().unwrap())?;
+ let schema = from_proto::parse_protobuf_file_scan_schema(
+ scan.base_conf.as_ref().unwrap(),
+ )?;
// Check if there's a projection and use projected schema for predicate parsing
let base_conf = scan.base_conf.as_ref().unwrap();
@@ -674,7 +673,7 @@ impl protobuf::PhysicalPlanNode {
.iter()
.map(|&i| schema.field(i as usize).clone())
.collect();
- Arc::new(Schema::new(projected_fields))
+ Arc::new(arrow::datatypes::Schema::new(projected_fields))
} else {
schema
};
@@ -691,7 +690,7 @@ impl protobuf::PhysicalPlanNode {
)
})
.transpose()?;
- let mut options = TableParquetOptions::default();
+ let mut options = datafusion_common::config::TableParquetOptions::default();
if let Some(table_options) = scan.parquet_options.as_ref() {
options = table_options.try_into()?;
@@ -1694,6 +1693,7 @@ impl protobuf::PhysicalPlanNode {
)))
}
+ #[cfg_attr(not(feature = "parquet"), expect(unused_variables))]
fn try_into_parquet_sink_physical_plan(
&self,
sink: &protobuf::ParquetSinkExecNode,
diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rs
index 19a76de3e5..7e29a1deae 100644
--- a/datafusion/proto/src/physical_plan/to_proto.rs
+++ b/datafusion/proto/src/physical_plan/to_proto.rs
@@ -20,32 +20,30 @@ use std::sync::Arc;
use arrow::array::RecordBatch;
use arrow::datatypes::Schema;
use arrow::ipc::writer::StreamWriter;
-#[cfg(feature = "parquet")]
-use datafusion::datasource::file_format::parquet::ParquetSink;
-use datafusion::datasource::physical_plan::FileSink;
-use datafusion::physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};
-use datafusion::physical_expr::ScalarFunctionExpr;
-use datafusion::physical_expr_common::physical_expr::snapshot_physical_expr;
-use datafusion::physical_expr_common::sort_expr::PhysicalSortExpr;
-use datafusion::physical_plan::expressions::{
- BinaryExpr, CaseExpr, CastExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr,
- Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,
-};
-use datafusion::physical_plan::udaf::AggregateFunctionExpr;
-use datafusion::physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};
-use datafusion::physical_plan::{Partitioning, PhysicalExpr, WindowExpr};
-use datafusion::{
- datasource::{
- file_format::{csv::CsvSink, json::JsonSink},
- listing::{FileRange, PartitionedFile},
- physical_plan::{FileScanConfig, FileSinkConfig},
- },
- physical_plan::expressions::LikeExpr,
-};
use datafusion_common::{
internal_datafusion_err, internal_err, not_impl_err, DataFusionError, Result,
};
+use datafusion_datasource::file_scan_config::FileScanConfig;
+use datafusion_datasource::file_sink_config::FileSink;
+use datafusion_datasource::file_sink_config::FileSinkConfig;
+use datafusion_datasource::{FileRange, PartitionedFile};
+use datafusion_datasource_csv::file_format::CsvSink;
+use datafusion_datasource_json::file_format::JsonSink;
+#[cfg(feature = "parquet")]
+use datafusion_datasource_parquet::file_format::ParquetSink;
use datafusion_expr::WindowFrame;
+use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};
+use datafusion_physical_expr::ScalarFunctionExpr;
+use datafusion_physical_expr_common::physical_expr::snapshot_physical_expr;
+use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
+use datafusion_physical_plan::expressions::LikeExpr;
+use datafusion_physical_plan::expressions::{
+ BinaryExpr, CaseExpr, CastExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr,
+ Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,
+};
+use datafusion_physical_plan::udaf::AggregateFunctionExpr;
+use datafusion_physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};
+use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};
use crate::protobuf::{
self, physical_aggregate_expr_node, physical_window_expr_node, PhysicalSortExprNode,
diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
index 3d51038eba..516f178cc0 100644
--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
@@ -71,6 +71,7 @@ use datafusion_common::{
internal_datafusion_err, internal_err, not_impl_err, plan_err, DFSchema, DFSchemaRef,
DataFusionError, Result, ScalarValue, TableReference,
};
+use datafusion_execution::TaskContext;
use datafusion_expr::dml::CopyTo;
use datafusion_expr::expr::{
self, Between, BinaryExpr, Case, Cast, GroupingSet, InList, Like, NullTreatment,
@@ -153,8 +154,11 @@ async fn roundtrip_logical_plan() -> Result<()> {
});
let extension_codec = TopKExtensionCodec {};
let bytes = logical_plan_to_bytes_with_extension_codec(&topk_plan, &extension_codec)?;
- let logical_round_trip =
- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &extension_codec)?;
+ let logical_round_trip = logical_plan_from_bytes_with_extension_codec(
+ &bytes,
+ &ctx.task_ctx(),
+ &extension_codec,
+ )?;
assert_eq!(format!("{topk_plan:?}"), format!("{logical_round_trip:?}"));
Ok(())
}
@@ -177,7 +181,7 @@ impl LogicalExtensionCodec for TestTableProviderCodec {
&self,
_buf: &[u8],
_inputs: &[LogicalPlan],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> Result<Extension> {
not_impl_err!("No extension codec provided")
}
@@ -191,7 +195,7 @@ impl LogicalExtensionCodec for TestTableProviderCodec {
buf: &[u8],
table_ref: &TableReference,
schema: SchemaRef,
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> Result<Arc<dyn TableProvider>> {
let msg = TestTableProto::decode(buf)
.map_err(|_| internal_datafusion_err!("Error decoding test table"))?;
@@ -240,7 +244,7 @@ async fn roundtrip_custom_tables() -> Result<()> {
let scan = ctx.table("t").await?.into_optimized_plan()?;
let bytes = logical_plan_to_bytes_with_extension_codec(&scan, &codec)?;
let logical_round_trip =
- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;
+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;
assert_eq!(format!("{scan:?}"), format!("{logical_round_trip:?}"));
Ok(())
}
@@ -266,7 +270,7 @@ async fn roundtrip_custom_memory_tables() -> Result<()> {
let plan = ctx.sql(query).await?.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));
Ok(())
@@ -293,7 +297,7 @@ async fn roundtrip_custom_listing_tables() -> Result<()> {
let plan = ctx.state().create_logical_plan(query).await?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
// Use exact matching to verify everything. Make sure during round-trip,
// information like constraints, column defaults, and other aspects of the plan are preserved.
assert_eq!(plan, logical_round_trip);
@@ -328,7 +332,7 @@ async fn roundtrip_logical_plan_aggregation_with_pk() -> Result<()> {
let plan = ctx.sql(query).await?.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
Ok(())
@@ -354,7 +358,7 @@ async fn roundtrip_logical_plan_aggregation() -> Result<()> {
let plan = ctx.sql(query).await?.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
Ok(())
@@ -380,7 +384,7 @@ async fn roundtrip_logical_plan_sort() -> Result<()> {
let plan = ctx.sql(query).await?.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
Ok(())
@@ -412,7 +416,7 @@ async fn roundtrip_logical_plan_dml() -> Result<()> {
for query in queries {
let plan = ctx.sql(query).await?.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(
format!("{plan}"),
format!("{logical_round_trip}"),
@@ -442,7 +446,7 @@ async fn roundtrip_logical_plan_copy_to_sql_options() -> Result<()> {
let codec = CsvLogicalExtensionCodec {};
let bytes = logical_plan_to_bytes_with_extension_codec(&plan, &codec)?;
let logical_round_trip =
- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;
+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
Ok(())
@@ -482,7 +486,7 @@ async fn roundtrip_logical_plan_copy_to_writer_options() -> Result<()> {
let codec = ParquetLogicalExtensionCodec {};
let bytes = logical_plan_to_bytes_with_extension_codec(&plan, &codec)?;
let logical_round_trip =
- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;
+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;
assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));
match logical_round_trip {
LogicalPlan::Copy(copy_to) => {
@@ -514,7 +518,7 @@ async fn roundtrip_logical_plan_copy_to_arrow() -> Result<()> {
let codec = ArrowLogicalExtensionCodec {};
let bytes = logical_plan_to_bytes_with_extension_codec(&plan, &codec)?;
let logical_round_trip =
- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;
+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
match logical_round_trip {
@@ -561,7 +565,7 @@ async fn roundtrip_logical_plan_copy_to_csv() -> Result<()> {
let codec = CsvLogicalExtensionCodec {};
let bytes = logical_plan_to_bytes_with_extension_codec(&plan, &codec)?;
let logical_round_trip =
- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;
+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;
assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));
match logical_round_trip {
@@ -628,7 +632,7 @@ async fn roundtrip_logical_plan_copy_to_json() -> Result<()> {
let codec = JsonLogicalExtensionCodec {};
let bytes = logical_plan_to_bytes_with_extension_codec(&plan, &codec)?;
let logical_round_trip =
- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;
+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
match logical_round_trip {
@@ -700,7 +704,7 @@ async fn roundtrip_logical_plan_copy_to_parquet() -> Result<()> {
let codec = ParquetLogicalExtensionCodec {};
let bytes = logical_plan_to_bytes_with_extension_codec(&plan, &codec)?;
let logical_round_trip =
- logical_plan_from_bytes_with_extension_codec(&bytes, &ctx, &codec)?;
+ logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
match logical_round_trip {
@@ -787,7 +791,7 @@ async fn roundtrip_logical_plan_distinct_on() -> Result<()> {
let plan = ctx.sql(query).await?.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
Ok(())
@@ -813,7 +817,7 @@ async fn roundtrip_single_count_distinct() -> Result<()> {
let plan = ctx.sql(query).await?.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
Ok(())
@@ -826,7 +830,7 @@ async fn roundtrip_logical_plan_with_extension() -> Result<()> {
.await?;
let plan = ctx.table("t1").await?.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
Ok(())
}
@@ -851,7 +855,7 @@ async fn roundtrip_logical_plan_unnest() -> Result<()> {
let query = "SELECT unnest(b) FROM t1";
let plan = ctx.sql(query).await?.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
Ok(())
}
@@ -1039,7 +1043,7 @@ async fn roundtrip_expr_api() -> Result<()> {
// ensure expressions created with the expr api can be round tripped
let plan = table.select(expr_list)?.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
Ok(())
}
@@ -1059,13 +1063,13 @@ async fn roundtrip_logical_plan_with_view_scan() -> Result<()> {
.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
// DROP
let plan = ctx.sql("DROP VIEW view_t1").await?.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(format!("{plan}"), format!("{logical_round_trip}"));
Ok(())
@@ -1160,7 +1164,7 @@ impl LogicalExtensionCodec for TopKExtensionCodec {
&self,
buf: &[u8],
inputs: &[LogicalPlan],
- ctx: &SessionContext,
+ ctx: &TaskContext,
) -> Result<Extension> {
if let Some((input, _)) = inputs.split_first() {
let proto = proto::TopKPlanProto::decode(buf).map_err(|e| {
@@ -1207,7 +1211,7 @@ impl LogicalExtensionCodec for TopKExtensionCodec {
_buf: &[u8],
_table_ref: &TableReference,
_schema: SchemaRef,
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> Result<Arc<dyn TableProvider>> {
internal_err!("unsupported plan type")
}
@@ -1230,7 +1234,7 @@ impl LogicalExtensionCodec for UDFExtensionCodec {
&self,
_buf: &[u8],
_inputs: &[LogicalPlan],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> Result<Extension> {
not_impl_err!("No extension codec provided")
}
@@ -1244,7 +1248,7 @@ impl LogicalExtensionCodec for UDFExtensionCodec {
_buf: &[u8],
_table_ref: &TableReference,
_schema: SchemaRef,
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> Result<Arc<dyn TableProvider>> {
internal_err!("unsupported plan type")
}
@@ -2269,7 +2273,7 @@ fn roundtrip_scalar_udf() {
&self,
_buf: &[u8],
_inputs: &[LogicalPlan],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> Result<Extension> {
not_impl_err!("LogicalExtensionCodec is not provided")
}
@@ -2283,7 +2287,7 @@ fn roundtrip_scalar_udf() {
_buf: &[u8],
_table_ref: &TableReference,
_schema: SchemaRef,
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> Result<Arc<dyn TableProvider>> {
not_impl_err!("LogicalExtensionCodec is not provided")
}
@@ -2677,7 +2681,7 @@ async fn roundtrip_recursive_query() {
let bytes = logical_plan_to_bytes(&plan).unwrap();
let ctx = SessionContext::new();
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx).unwrap();
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx()).unwrap();
assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));
let dataframe = ctx.execute_logical_plan(logical_round_trip).await.unwrap();
let output_round_trip = dataframe.collect().await.unwrap();
@@ -2708,7 +2712,7 @@ async fn roundtrip_union_query() -> Result<()> {
.await?;
ctx.register_csv("t2", "tests/testdata/test.csv", CsvReadOptions::default())
.await?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
// proto deserialization only supports 2-way union, hence this plan has nested unions
// apply the flatten unions optimizer rule to be able to compare
let optimizer = Optimizer::with_rules(vec![Arc::new(EliminateNestedUnion::new())]);
@@ -2745,7 +2749,7 @@ async fn roundtrip_custom_listing_tables_schema() -> Result<()> {
.clone();
let bytes = logical_plan_to_bytes(&plan)?;
- let new_plan = logical_plan_from_bytes(&bytes, &ctx)?;
+ let new_plan = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(plan, new_plan);
Ok(())
}
@@ -2782,7 +2786,7 @@ async fn roundtrip_custom_listing_tables_schema_table_scan_projection() -> Resul
.build()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let new_plan = logical_plan_from_bytes(&bytes, &ctx)?;
+ let new_plan = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(plan, new_plan);
Ok(())
@@ -2796,7 +2800,7 @@ async fn roundtrip_arrow_scan() -> Result<()> {
.await?
.into_optimized_plan()?;
let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));
Ok(())
}
diff --git a/datafusion/proto/tests/cases/serialize.rs b/datafusion/proto/tests/cases/serialize.rs
index c9ef4377d4..3d69183668 100644
--- a/datafusion/proto/tests/cases/serialize.rs
+++ b/datafusion/proto/tests/cases/serialize.rs
@@ -61,7 +61,7 @@ fn json_to_plan() {
let input = r#"{"emptyRelation":{}}"#.to_string();
let ctx = SessionContext::new();
- let actual = logical_plan_from_json(&input, &ctx).unwrap();
+ let actual = logical_plan_from_json(&input, &ctx.task_ctx()).unwrap();
let result = matches!(actual, LogicalPlan::EmptyRelation(_));
assert!(result, "Should parse empty relation");
}
diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.md
index 0b9da1b5a8..8b03193e7f 100644
--- a/docs/source/library-user-guide/upgrading.md
+++ b/docs/source/library-user-guide/upgrading.md
@@ -94,6 +94,28 @@ has changed from `&str` to `&Dialect`.
crate under the `config` module that provides type safety
and better validation for SQL dialect selection
+### Reorganization of `ListingTable` into `datafusion-catalog-listing` crate
+
+There has been a long standing request to remove features such as `ListingTable`
+from the `datafusion` crate to support faster build times. The structs
+`ListingOptions`, `ListingTable`, and `ListingTableConfig` are now available
+within the `datafusion-catalog-listing` crate. These are re-exported in
+the `datafusion` crate, so this should be a minimal impact to existing users.
+
+See [issue #14462] and [issue #17713] for more details.
+
+[issue #14462]: https://github.com/apache/datafusion/issues/14462
+[issue #17713]: https://github.com/apache/datafusion/issues/17713
+
+### Reorganization of `ArrowSource` into `datafusion-datasource-arrow` crate
+
+To support [issue #17713] the `ArrowSource` code has been removed from
+the `datafusion` core crate into it's own crate, `datafusion-datasource-arrow`.
+This follows the pattern for the AVRO, CSV, JSON, and Parquet data sources.
+Users may need to update their paths to account for these changes.
+
+See [issue #17713] for more details.
+
## DataFusion `50.0.0`
### ListingTable automatically detects Hive Partitioned tables