Apache DataFusion SQL Query Engine
fix(proto): preserve Arrow IPC stream format (#24224)
Burak Şenf956fd1128d1
diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rsindex dba99a9758..dd973c03ae 100644--- a/datafusion/datasource-arrow/src/source.rs+++ b/datafusion/datasource-arrow/src/source.rs@@ -403,10 +403,8 @@ impl FileSource for ArrowSource {datafusion_physical_plan::apply_expression_roots(self.projection.source.iter(), f)}- /// Emit an `ArrowScan` node wrapping the shared base config.- ///- /// Decoding defaults to the IPC file format because protobuf does not- /// distinguish it from the IPC stream format.+ /// Emit an `ArrowScan` node wrapping the shared base config and recording+ /// which Arrow IPC format (file or stream) this source reads.#[cfg(feature = "proto")]fn try_to_proto(&self,@@ -416,10 +414,16 @@ impl FileSource for ArrowSource {use datafusion_proto_models::protobuf;use protobuf::physical_plan_node::PhysicalPlanType;+ let format = match self.format {+ ArrowFormat::File => protobuf::ArrowIpcFormat::File,+ ArrowFormat::Stream => protobuf::ArrowIpcFormat::Stream,+ };+Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::ArrowScan(protobuf::ArrowScanExecNode {base_conf: Some(base.try_to_proto(ctx)?),+ format: format as i32,},)),}))@@ -430,8 +434,8 @@ impl FileSource for ArrowSource {impl ArrowSource {/// Reconstructs a `DataSourceExec` from a protobuf `ArrowScan`.///- /// Defaults to the IPC file format because protobuf does not distinguish it- /// from the IPC stream format.+ /// Payloads encoded before the `format` field existed leave it unset;+ /// those decode as the IPC file format, matching the historical behavior.pub fn try_from_proto(node: &datafusion_proto_models::protobuf::PhysicalPlanNode,ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,@@ -455,9 +459,21 @@ impl ArrowSource {)})?;+ let format = protobuf::ArrowIpcFormat::try_from(scan.format).map_err(|_| {+ datafusion_common::internal_datafusion_err!(+ "Unknown ArrowIpcFormat: {}",+ scan.format+ )+ })?;+let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;- let source = Arc::new(ArrowSource::new_file_source(table_schema));- let scan_conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?;+ let source = match format {+ protobuf::ArrowIpcFormat::Stream => {+ ArrowSource::new_stream_file_source(table_schema)+ }+ protobuf::ArrowIpcFormat::File => ArrowSource::new_file_source(table_schema),+ };+ let scan_conf = FileScanConfig::try_from_proto(base_conf, ctx, Arc::new(source))?;Ok(DataSourceExec::from_data_source(scan_conf))}}diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex 43a90264c2..f29d63c476 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -1296,8 +1296,18 @@ message AvroScanExecNode {FileScanExecConf base_conf = 1;}+// Identifies which Arrow IPC format an ArrowScanExecNode reads.+enum ArrowIpcFormat {+ // Arrow IPC file format (with footer, supports range-based parallel reading).+ // This is the default for payloads encoded before the format field existed.+ ARROW_IPC_FORMAT_FILE = 0;+ // Arrow IPC stream format (without footer, sequential reading only)+ ARROW_IPC_FORMAT_STREAM = 1;+}+message ArrowScanExecNode {FileScanExecConf base_conf = 1;+ ArrowIpcFormat format = 2;}message MemoryScanExecNode {diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex 908f9752b7..19121b2dbe 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -1445,6 +1445,77 @@ impl<'de> serde::Deserialize<'de> for AnalyzedLogicalPlanType {deserializer.deserialize_struct("datafusion.AnalyzedLogicalPlanType", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for ArrowIpcFormat {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ let variant = match self {+ Self::File => "ARROW_IPC_FORMAT_FILE",+ Self::Stream => "ARROW_IPC_FORMAT_STREAM",+ };+ serializer.serialize_str(variant)+ }+}+impl<'de> serde::Deserialize<'de> for ArrowIpcFormat {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "ARROW_IPC_FORMAT_FILE",+ "ARROW_IPC_FORMAT_STREAM",+ ];++ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = ArrowIpcFormat;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ fn visit_i64<E>(self, v: i64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Signed(v), &self)+ })+ }++ fn visit_u64<E>(self, v: u64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Unsigned(v), &self)+ })+ }++ fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ match value {+ "ARROW_IPC_FORMAT_FILE" => Ok(ArrowIpcFormat::File),+ "ARROW_IPC_FORMAT_STREAM" => Ok(ArrowIpcFormat::Stream),+ _ => Err(serde::de::Error::unknown_variant(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_any(GeneratedVisitor)+ }+}impl serde::Serialize for ArrowScanExecNode {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>@@ -1456,10 +1527,18 @@ impl serde::Serialize for ArrowScanExecNode {if self.base_conf.is_some() {len += 1;}+ if self.format != 0 {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.ArrowScanExecNode", len)?;if let Some(v) = self.base_conf.as_ref() {struct_ser.serialize_field("baseConf", v)?;}+ if self.format != 0 {+ let v = ArrowIpcFormat::try_from(self.format)+ .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.format)))?;+ struct_ser.serialize_field("format", &v)?;+ }struct_ser.end()}}@@ -1472,11 +1551,13 @@ impl<'de> serde::Deserialize<'de> for ArrowScanExecNode {const FIELDS: &[&str] = &["base_conf","baseConf",+ "format",];#[allow(clippy::enum_variant_names)]enum GeneratedField {BaseConf,+ Format,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -1499,6 +1580,7 @@ impl<'de> serde::Deserialize<'de> for ArrowScanExecNode {{match value {"baseConf" | "base_conf" => Ok(GeneratedField::BaseConf),+ "format" => Ok(GeneratedField::Format),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -1519,6 +1601,7 @@ impl<'de> serde::Deserialize<'de> for ArrowScanExecNode {V: serde::de::MapAccess<'de>,{let mut base_conf__ = None;+ let mut format__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::BaseConf => {@@ -1527,10 +1610,17 @@ impl<'de> serde::Deserialize<'de> for ArrowScanExecNode {}base_conf__ = map_.next_value()?;}+ GeneratedField::Format => {+ if format__.is_some() {+ return Err(serde::de::Error::duplicate_field("format"));+ }+ format__ = Some(map_.next_value::<ArrowIpcFormat>()? as i32);+ }}}Ok(ArrowScanExecNode {base_conf: base_conf__,+ format: format__.unwrap_or_default(),})}}diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex ba00577ab9..a6fe825ff6 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -1996,6 +1996,8 @@ pub struct AvroScanExecNode {pub struct ArrowScanExecNode {#[prost(message, optional, tag = "1")]pub base_conf: ::core::option::Option<FileScanExecConf>,+ #[prost(enumeration = "ArrowIpcFormat", tag = "2")]+ pub format: i32,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct MemoryScanExecNode {@@ -2794,6 +2796,36 @@ impl InsertOp {}}}+/// Identifies which Arrow IPC format an ArrowScanExecNode reads.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]+#[repr(i32)]+pub enum ArrowIpcFormat {+ /// Arrow IPC file format (with footer, supports range-based parallel reading).+ /// This is the default for payloads encoded before the format field existed.+ File = 0,+ /// Arrow IPC stream format (without footer, sequential reading only)+ Stream = 1,+}+impl ArrowIpcFormat {+ /// String value of the enum field names used in the ProtoBuf definition.+ ///+ /// The values are not transformed in any way and thus are considered stable+ /// (if the ProtoBuf definition does not change) and safe for programmatic use.+ pub fn as_str_name(&self) -> &'static str {+ match self {+ Self::File => "ARROW_IPC_FORMAT_FILE",+ Self::Stream => "ARROW_IPC_FORMAT_STREAM",+ }+ }+ /// Creates an enum from field names used in the ProtoBuf definition.+ pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {+ match value {+ "ARROW_IPC_FORMAT_FILE" => Some(Self::File),+ "ARROW_IPC_FORMAT_STREAM" => Some(Self::Stream),+ _ => None,+ }+ }+}#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]#[repr(i32)]pub enum PartitionMode {diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rsindex 04708dec64..5cdd42a0ac 100644--- a/datafusion/proto/tests/cases/plans/sources.rs+++ b/datafusion/proto/tests/cases/plans/sources.rs@@ -57,10 +57,15 @@ use datafusion_datasource::{TableSchema, TableSchemaBuilder};use datafusion_expr::ColumnarValue;use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;+use datafusion_proto::bytes::{+ physical_plan_from_bytes_with_proto_converter,+ physical_plan_to_bytes_with_proto_converter,+};use datafusion_proto::physical_plan::{AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,PhysicalExtensionCodec, PhysicalProtoConverterExtension,};+use datafusion_proto::protobuf;use datafusion_proto::protobuf::PhysicalPlanNode;use prost::Message;use std::collections::HashMap;@@ -156,6 +161,22 @@ fn roundtrip_parquet_exec_attaches_cached_reader_factory_after_roundtrip() -> ReOk(())}+/// Returns `FileSource::file_type` of a `DataSourceExec` file scan, e.g.+/// "arrow" vs "arrow_stream". The two Arrow IPC formats print identically in+/// plan debug output, so roundtrip tests must inspect the source directly.+fn scan_file_type(plan: &Arc<dyn ExecutionPlan>) -> Result<String> {+ let data_source = plan.downcast_ref::<DataSourceExec>().ok_or_else(|| {+ internal_datafusion_err!("Expected DataSourceExec after roundtrip")+ })?;+ let file_scan = data_source+ .data_source()+ .downcast_ref::<FileScanConfig>()+ .ok_or_else(|| {+ internal_datafusion_err!("Expected FileScanConfig after roundtrip")+ })?;+ Ok(file_scan.file_source().file_type().to_string())+}+#[test]fn roundtrip_arrow_scan() -> Result<()> {let file_schema =@@ -177,7 +198,85 @@ fn roundtrip_arrow_scan() -> Result<()> {}).build();- roundtrip_test(DataSourceExec::from_data_source(scan_config))+ let roundtripped = roundtrip_test_and_return(+ DataSourceExec::from_data_source(scan_config),+ &SessionContext::new(),+ &DefaultPhysicalExtensionCodec {},+ &DefaultPhysicalProtoConverter {},+ )?;+ assert_eq!(scan_file_type(&roundtripped)?, "arrow");+ Ok(())+}++#[test]+fn roundtrip_arrow_stream_scan() -> Result<()> {+ let file_schema =+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));+ let file_source = Arc::new(ArrowSource::new_stream_file_source(TableSchema::from(+ &file_schema,+ )));+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.arrows".to_string(),+ 1024,+ )])])+ .build();++ let roundtripped = roundtrip_test_and_return(+ DataSourceExec::from_data_source(scan_config),+ &SessionContext::new(),+ &DefaultPhysicalExtensionCodec {},+ &DefaultPhysicalProtoConverter {},+ )?;+ assert_eq!(scan_file_type(&roundtripped)?, "arrow_stream");+ Ok(())+}++#[test]+fn arrow_scan_without_format_field_decodes_as_file_format() -> Result<()> {+ // Payloads encoded before `ArrowScanExecNode.format` existed carry no+ // format discriminator; they must keep decoding as the IPC file format.+ let file_schema =+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));+ let file_source = Arc::new(ArrowSource::new_file_source(TableSchema::from(+ &file_schema,+ )));+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.arrow".to_string(),+ 1024,+ )])])+ .build();++ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let bytes = physical_plan_to_bytes_with_proto_converter(+ DataSourceExec::from_data_source(scan_config),+ &codec,+ &proto_converter,+ )?;++ let mut node = PhysicalPlanNode::decode(bytes.as_ref()).map_err(|e| {+ internal_datafusion_err!("Failed to decode PhysicalPlanNode: {e}")+ })?;+ match node.physical_plan_type.as_mut() {+ Some(protobuf::physical_plan_node::PhysicalPlanType::ArrowScan(scan)) => {+ scan.format = protobuf::ArrowIpcFormat::File as i32;+ }+ other => return internal_err!("Expected ArrowScan node, got {other:?}"),+ }++ let ctx = SessionContext::new();+ let decoded = physical_plan_from_bytes_with_proto_converter(+ &node.encode_to_vec(),+ ctx.task_ctx().as_ref(),+ &codec,+ &proto_converter,+ )?;+ assert_eq!(scan_file_type(&decoded)?, "arrow");+ Ok(())}#[test]