Apache DataFusion SQL Query Engine
fix: regression of `dict_id` in physical plan proto (#20063)
Kumar Ujjawal8ba3d2617b49
diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex af427ef5ab..967bda627f 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -28,7 +28,12 @@ use arrow::datatypes::{DataType, Field, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, Schema,TimeUnit, UnionFields, UnionMode, i256,};-use arrow::ipc::{reader::read_record_batch, root_as_message};+use arrow::ipc::{+ convert::fb_to_schema,+ reader::{read_dictionary, read_record_batch},+ root_as_message,+ writer::{DictionaryTracker, IpcDataGenerator, IpcWriteOptions},+};use datafusion_common::{Column, ColumnStatistics, Constraint, Constraints, DFSchema, DFSchemaRef,@@ -397,7 +402,7 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue {Value::Float32Value(v) => Self::Float32(Some(*v)),Value::Float64Value(v) => Self::Float64(Some(*v)),Value::Date32Value(v) => Self::Date32(Some(*v)),- // ScalarValue::List is serialized using arrow IPC format+ // Nested ScalarValue types are serialized using arrow IPC formatValue::ListValue(v)| Value::FixedSizeListValue(v)| Value::LargeListValue(v)@@ -414,55 +419,83 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue {schema_ref.try_into()?} else {return Err(Error::General(- "Invalid schema while deserializing ScalarValue::List"+ "Invalid schema while deserializing nested ScalarValue".to_string(),));};+ // IPC dictionary batch IDs are assigned when encoding the schema, but our protobuf+ // `Schema` doesn't preserve those IDs. Reconstruct them deterministically by+ // round-tripping the schema through IPC.+ let schema: Schema = {+ let ipc_gen = IpcDataGenerator {};+ let write_options = IpcWriteOptions::default();+ let mut dict_tracker = DictionaryTracker::new(false);+ let encoded_schema = ipc_gen.schema_to_bytes_with_dictionary_tracker(+ &schema,+ &mut dict_tracker,+ &write_options,+ );+ let message =+ root_as_message(encoded_schema.ipc_message.as_slice()).map_err(+ |e| {+ Error::General(format!(+ "Error IPC schema message while deserializing nested ScalarValue: {e}"+ ))+ },+ )?;+ let ipc_schema = message.header_as_schema().ok_or_else(|| {+ Error::General(+ "Unexpected message type deserializing nested ScalarValue schema"+ .to_string(),+ )+ })?;+ fb_to_schema(ipc_schema)+ };+let message = root_as_message(ipc_message.as_slice()).map_err(|e| {Error::General(format!(- "Error IPC message while deserializing ScalarValue::List: {e}"+ "Error IPC message while deserializing nested ScalarValue: {e}"))})?;let buffer = Buffer::from(arrow_data.as_slice());let ipc_batch = message.header_as_record_batch().ok_or_else(|| {Error::General(- "Unexpected message type deserializing ScalarValue::List"+ "Unexpected message type deserializing nested ScalarValue".to_string(),)})?;- let dict_by_id: HashMap<i64,ArrayRef> = dictionaries.iter().map(|protobuf::scalar_nested_value::Dictionary { ipc_message, arrow_data }| {+ let mut dict_by_id: HashMap<i64, ArrayRef> = HashMap::new();+ for protobuf::scalar_nested_value::Dictionary {+ ipc_message,+ arrow_data,+ } in dictionaries+ {let message = root_as_message(ipc_message.as_slice()).map_err(|e| {Error::General(format!(- "Error IPC message while deserializing ScalarValue::List dictionary message: {e}"+ "Error IPC message while deserializing nested ScalarValue dictionary message: {e}"))})?;let buffer = Buffer::from(arrow_data.as_slice());let dict_batch = message.header_as_dictionary_batch().ok_or_else(|| {Error::General(- "Unexpected message type deserializing ScalarValue::List dictionary message"+ "Unexpected message type deserializing nested ScalarValue dictionary message".to_string(),)})?;-- let id = dict_batch.id();-- let record_batch = read_record_batch(+ read_dictionary(&buffer,- dict_batch.data().unwrap(),- Arc::new(schema.clone()),- &Default::default(),- None,+ dict_batch,+ &schema,+ &mut dict_by_id,&message.version(),- )?;-- let values: ArrayRef = Arc::clone(record_batch.column(0));-- Ok((id, values))- }).collect::<datafusion_common::Result<HashMap<_, _>>>()?;+ )+ .map_err(|e| arrow_datafusion_err!(e))+ .map_err(|e| e.context("Decoding nested ScalarValue dictionary"))?;+ }let record_batch = read_record_batch(&buffer,@@ -473,7 +506,7 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue {&message.version(),).map_err(|e| arrow_datafusion_err!(e))- .map_err(|e| e.context("Decoding ScalarValue::List Value"))?;+ .map_err(|e| e.context("Decoding nested ScalarValue value"))?;let arr = record_batch.column(0);match value {Value::ListValue(_) => {diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rsindex db405b29a5..01b671e37d 100644--- a/datafusion/proto-common/src/to_proto/mod.rs+++ b/datafusion/proto-common/src/to_proto/mod.rs@@ -1031,7 +1031,7 @@ fn create_proto_scalar<I, T: FnOnce(&I) -> protobuf::scalar_value::Value>(Ok(protobuf::ScalarValue { value: Some(value) })}-// ScalarValue::List / FixedSizeList / LargeList / Struct / Map are serialized using+// Nested ScalarValue types (List / FixedSizeList / LargeList / Struct / Map) are serialized using// Arrow IPC messages as a single column RecordBatchfn encode_scalar_nested_value(arr: ArrayRef,@@ -1039,13 +1039,20 @@ fn encode_scalar_nested_value() -> Result<protobuf::ScalarValue, Error> {let batch = RecordBatch::try_from_iter(vec![("field_name", arr)]).map_err(|e| {Error::General(format!(- "Error creating temporary batch while encoding ScalarValue::List: {e}"+ "Error creating temporary batch while encoding nested ScalarValue: {e}"))})?;let ipc_gen = IpcDataGenerator {};let mut dict_tracker = DictionaryTracker::new(false);let write_options = IpcWriteOptions::default();+ // The IPC writer requires pre-allocated dictionary IDs (normally assigned when+ // serializing the schema). Populate `dict_tracker` by encoding the schema first.+ ipc_gen.schema_to_bytes_with_dictionary_tracker(+ batch.schema().as_ref(),+ &mut dict_tracker,+ &write_options,+ );let mut compression_context = CompressionContext::default();let (encoded_dictionaries, encoded_message) = ipc_gen.encode(@@ -1055,7 +1062,7 @@ fn encode_scalar_nested_value(&mut compression_context,).map_err(|e| {- Error::General(format!("Error encoding ScalarValue::List as IPC: {e}"))+ Error::General(format!("Error encoding nested ScalarValue as IPC: {e}"))})?;let schema: protobuf::Schema = batch.schema().try_into()?;diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex 0a26025a3c..fd4de81140 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -2566,6 +2566,25 @@ fn custom_proto_converter_intercepts() -> Result<()> {Ok(())}+#[test]+fn roundtrip_call_null_scalar_struct_dict() -> Result<()> {+ let data_type = DataType::Struct(Fields::from(vec![Field::new(+ "item",+ DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),+ true,+ )]));++ let schema = Arc::new(Schema::new(vec![Field::new("a", data_type.clone(), true)]));+ let scan = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ let scalar = lit(ScalarValue::try_from(data_type)?);+ let filter = Arc::new(FilterExec::try_new(+ Arc::new(BinaryExpr::new(scalar, Operator::Eq, col("a", &schema)?)),+ scan,+ )?);++ roundtrip_test(filter)+}+/// Test that expression deduplication works during deserialization./// When the same expression Arc is serialized multiple times, it should be/// deduplicated on deserialization (sharing the same Arc).