Skip to content

Apache DataFusion SQL Query Engine

Add protoc support for ArrowScanExecNode (#20280) (#20284)

Josh Elkind5fccac12cc6c
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto
index 67c6d5ae16..7c02688676 100644
--- a/datafusion/proto/proto/datafusion.proto
+++ b/datafusion/proto/proto/datafusion.proto
@@ -751,6 +751,7 @@ message PhysicalPlanNode {
MemoryScanExecNode memory_scan = 35;
AsyncFuncExecNode async_func = 36;
BufferExecNode buffer = 37;
+ ArrowScanExecNode arrow_scan = 38;
}
}
@@ -1106,6 +1107,10 @@ message AvroScanExecNode {
FileScanExecConf base_conf = 1;
}
+message ArrowScanExecNode {
+ FileScanExecConf base_conf = 1;
+}
+
message MemoryScanExecNode {
repeated bytes partitions = 1;
datafusion_common.Schema schema = 2;
diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs
index b77060394f..5b2b9133ce 100644
--- a/datafusion/proto/src/generated/pbjson.rs
+++ b/datafusion/proto/src/generated/pbjson.rs
@@ -1298,6 +1298,98 @@ impl<'de> serde::Deserialize<'de> for AnalyzedLogicalPlanType {
deserializer.deserialize_struct("datafusion.AnalyzedLogicalPlanType", FIELDS, GeneratedVisitor)
}
}
+impl serde::Serialize for ArrowScanExecNode {
+ #[allow(deprecated)]
+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ use serde::ser::SerializeStruct;
+ let mut len = 0;
+ if self.base_conf.is_some() {
+ len += 1;
+ }
+ let mut struct_ser = serializer.serialize_struct("datafusion.ArrowScanExecNode", len)?;
+ if let Some(v) = self.base_conf.as_ref() {
+ struct_ser.serialize_field("baseConf", v)?;
+ }
+ struct_ser.end()
+ }
+}
+impl<'de> serde::Deserialize<'de> for ArrowScanExecNode {
+ #[allow(deprecated)]
+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ const FIELDS: &[&str] = &[
+ "base_conf",
+ "baseConf",
+ ];
+
+ #[allow(clippy::enum_variant_names)]
+ enum GeneratedField {
+ BaseConf,
+ }
+ impl<'de> serde::Deserialize<'de> for GeneratedField {
+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ struct GeneratedVisitor;
+
+ impl serde::de::Visitor<'_> for GeneratedVisitor {
+ type Value = GeneratedField;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(formatter, "expected one of: {:?}", &FIELDS)
+ }
+
+ #[allow(unused_variables)]
+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>
+ where
+ E: serde::de::Error,
+ {
+ match value {
+ "baseConf" | "base_conf" => Ok(GeneratedField::BaseConf),
+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),
+ }
+ }
+ }
+ deserializer.deserialize_identifier(GeneratedVisitor)
+ }
+ }
+ struct GeneratedVisitor;
+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {
+ type Value = ArrowScanExecNode;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ formatter.write_str("struct datafusion.ArrowScanExecNode")
+ }
+
+ fn visit_map<V>(self, mut map_: V) -> std::result::Result<ArrowScanExecNode, V::Error>
+ where
+ V: serde::de::MapAccess<'de>,
+ {
+ let mut base_conf__ = None;
+ while let Some(k) = map_.next_key()? {
+ match k {
+ GeneratedField::BaseConf => {
+ if base_conf__.is_some() {
+ return Err(serde::de::Error::duplicate_field("baseConf"));
+ }
+ base_conf__ = map_.next_value()?;
+ }
+ }
+ }
+ Ok(ArrowScanExecNode {
+ base_conf: base_conf__,
+ })
+ }
+ }
+ deserializer.deserialize_struct("datafusion.ArrowScanExecNode", FIELDS, GeneratedVisitor)
+ }
+}
impl serde::Serialize for AsyncFuncExecNode {
#[allow(deprecated)]
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
@@ -17783,6 +17875,9 @@ impl serde::Serialize for PhysicalPlanNode {
physical_plan_node::PhysicalPlanType::Buffer(v) => {
struct_ser.serialize_field("buffer", v)?;
}
+ physical_plan_node::PhysicalPlanType::ArrowScan(v) => {
+ struct_ser.serialize_field("arrowScan", v)?;
+ }
}
}
struct_ser.end()
@@ -17851,6 +17946,8 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {
"async_func",
"asyncFunc",
"buffer",
+ "arrow_scan",
+ "arrowScan",
];
#[allow(clippy::enum_variant_names)]
@@ -17891,6 +17988,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {
MemoryScan,
AsyncFunc,
Buffer,
+ ArrowScan,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -17948,6 +18046,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {
"memoryScan" | "memory_scan" => Ok(GeneratedField::MemoryScan),
"asyncFunc" | "async_func" => Ok(GeneratedField::AsyncFunc),
"buffer" => Ok(GeneratedField::Buffer),
+ "arrowScan" | "arrow_scan" => Ok(GeneratedField::ArrowScan),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -18220,6 +18319,13 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {
return Err(serde::de::Error::duplicate_field("buffer"));
}
physical_plan_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_plan_node::PhysicalPlanType::Buffer)
+;
+ }
+ GeneratedField::ArrowScan => {
+ if physical_plan_type__.is_some() {
+ return Err(serde::de::Error::duplicate_field("arrowScan"));
+ }
+ physical_plan_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_plan_node::PhysicalPlanType::ArrowScan)
;
}
}
diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs
index e95cddcc2c..d9602665c2 100644
--- a/datafusion/proto/src/generated/prost.rs
+++ b/datafusion/proto/src/generated/prost.rs
@@ -1079,7 +1079,7 @@ pub mod table_reference {
pub struct PhysicalPlanNode {
#[prost(
oneof = "physical_plan_node::PhysicalPlanType",
- tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37"
+ tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38"
)]
pub physical_plan_type: ::core::option::Option<physical_plan_node::PhysicalPlanType>,
}
@@ -1161,6 +1161,8 @@ pub mod physical_plan_node {
AsyncFunc(::prost::alloc::boxed::Box<super::AsyncFuncExecNode>),
#[prost(message, tag = "37")]
Buffer(::prost::alloc::boxed::Box<super::BufferExecNode>),
+ #[prost(message, tag = "38")]
+ ArrowScan(super::ArrowScanExecNode),
}
}
#[derive(Clone, PartialEq, ::prost::Message)]
@@ -1669,6 +1671,11 @@ pub struct AvroScanExecNode {
pub base_conf: ::core::option::Option<FileScanExecConf>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ArrowScanExecNode {
+ #[prost(message, optional, tag = "1")]
+ pub base_conf: ::core::option::Option<FileScanExecConf>,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
pub struct MemoryScanExecNode {
#[prost(bytes = "vec", repeated, tag = "1")]
pub partitions: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec<u8>>,
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index 60d8b5705b..bfba715b91 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -34,6 +34,7 @@ use datafusion_datasource::file_compression_type::FileCompressionType;
use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
use datafusion_datasource::sink::DataSinkExec;
use datafusion_datasource::source::{DataSource, DataSourceExec};
+use datafusion_datasource_arrow::source::ArrowSource;
#[cfg(feature = "avro")]
use datafusion_datasource_avro::source::AvroSource;
use datafusion_datasource_csv::file_format::CsvSink;
@@ -199,6 +200,9 @@ impl protobuf::PhysicalPlanNode {
PhysicalPlanType::MemoryScan(scan) => {
self.try_into_memory_scan_physical_plan(scan, ctx, codec, proto_converter)
}
+ PhysicalPlanType::ArrowScan(scan) => {
+ self.try_into_arrow_scan_physical_plan(scan, ctx, codec, proto_converter)
+ }
PhysicalPlanType::CoalesceBatches(coalesce_batches) => self
.try_into_coalesce_batches_physical_plan(
coalesce_batches,
@@ -774,6 +778,27 @@ impl protobuf::PhysicalPlanNode {
Ok(DataSourceExec::from_data_source(scan_conf))
}
+ fn try_into_arrow_scan_physical_plan(
+ &self,
+ scan: &protobuf::ArrowScanExecNode,
+ ctx: &TaskContext,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
+ internal_datafusion_err!("base_conf in ArrowScanExecNode is missing.")
+ })?;
+ let table_schema = parse_table_schema_from_proto(base_conf)?;
+ let scan_conf = parse_protobuf_file_scan_config(
+ base_conf,
+ ctx,
+ codec,
+ proto_converter,
+ Arc::new(ArrowSource::new_file_source(table_schema)),
+ )?;
+ Ok(DataSourceExec::from_data_source(scan_conf))
+ }
+
#[cfg_attr(not(feature = "parquet"), expect(unused_variables))]
fn try_into_parquet_scan_physical_plan(
&self,
@@ -2867,6 +2892,23 @@ impl protobuf::PhysicalPlanNode {
}
}
+ if let Some(scan_conf) = data_source.as_any().downcast_ref::<FileScanConfig>() {
+ let source = scan_conf.file_source();
+ if let Some(_arrow_source) = source.as_any().downcast_ref::<ArrowSource>() {
+ return Ok(Some(protobuf::PhysicalPlanNode {
+ physical_plan_type: Some(PhysicalPlanType::ArrowScan(
+ protobuf::ArrowScanExecNode {
+ base_conf: Some(serialize_file_scan_config(
+ scan_conf,
+ codec,
+ proto_converter,
+ )?),
+ },
+ )),
+ }));
+ }
+ }
+
#[cfg(feature = "parquet")]
if let Some((maybe_parquet, conf)) =
data_source_exec.downcast_to_file_source::<ParquetSource>()
diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
index fd4de81140..bc310150d8 100644
--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
@@ -36,8 +36,8 @@ use datafusion::datasource::listing::{
};
use datafusion::datasource::object_store::ObjectStoreUrl;
use datafusion::datasource::physical_plan::{
- FileGroup, FileOutputMode, FileScanConfigBuilder, FileSinkConfig, ParquetSource,
- wrap_partition_type_in_dict, wrap_partition_value_in_dict,
+ ArrowSource, FileGroup, FileOutputMode, FileScanConfigBuilder, FileSinkConfig,
+ ParquetSource, wrap_partition_type_in_dict, wrap_partition_value_in_dict,
};
use datafusion::datasource::sink::DataSinkExec;
use datafusion::datasource::source::DataSourceExec;
@@ -929,6 +929,30 @@ fn roundtrip_parquet_exec_with_pruning_predicate() -> Result<()> {
roundtrip_test(DataSourceExec::from_data_source(scan_config))
}
+#[test]
+fn roundtrip_arrow_scan() -> Result<()> {
+ let file_schema =
+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
+
+ let table_schema = TableSchema::new(file_schema.clone(), vec![]);
+ let file_source = Arc::new(ArrowSource::new_file_source(table_schema));
+
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.arrow".to_string(),
+ 1024,
+ )])])
+ .with_statistics(Statistics {
+ num_rows: Precision::Inexact(100),
+ total_byte_size: Precision::Inexact(1024),
+ column_statistics: Statistics::unknown_column(&file_schema),
+ })
+ .build();
+
+ roundtrip_test(DataSourceExec::from_data_source(scan_config))
+}
+
#[tokio::test]
async fn roundtrip_parquet_exec_with_table_partition_cols() -> Result<()> {
let mut file_group =