Apache DataFusion SQL Query Engine
fix: preserve dotted column relation qualifiers (#25334)
Anurag Tryambak Raut7b1e55d4c229
diff --git a/datafusion/proto-common/proto/datafusion_common.proto b/datafusion/proto-common/proto/datafusion_common.protoindex f4ef1e6a2c..c3056a97ff 100644--- a/datafusion/proto-common/proto/datafusion_common.proto+++ b/datafusion/proto-common/proto/datafusion_common.proto@@ -22,6 +22,7 @@ package datafusion_common;message ColumnRelation {string relation = 1;+ repeated string parts = 2;}message Column {diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex 8984e78452..4c2fdc9fe5 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -147,7 +147,14 @@ whereimpl From<protobuf::ColumnRelation> for TableReference {fn from(rel: protobuf::ColumnRelation) -> Self {- Self::parse_str_normalized(rel.relation.as_str(), true)+ match rel.parts.as_slice() {+ [table] => Self::bare(table.as_str()),+ [schema, table] => Self::partial(schema.as_str(), table.as_str()),+ [catalog, schema, table] => {+ Self::full(catalog.as_str(), schema.as_str(), table.as_str())+ }+ _ => Self::parse_str_normalized(rel.relation.as_str(), true),+ }}}@@ -1440,6 +1447,7 @@ where#[cfg(test)]mod tests {+ use datafusion_common::TableReference;use datafusion_common::config::{MaxRowGroupBytes, ParquetCdcOptions, ParquetOptions, TableParquetOptions,};@@ -1461,6 +1469,121 @@ mod tests {);}+ #[test]+ fn column_relation_round_trip_preserves_dotted_bare_table() {+ let column = datafusion_common::Column::new(+ Some(TableReference::bare("has.dot")),+ "column",+ );++ let proto: crate::protobuf_common::Column = (&column).into();+ let relation = proto.relation.expect("relation should be present");++ assert_eq!(relation.relation, "has.dot");+ assert_eq!(relation.parts, vec!["has.dot".to_string()]);++ let recovered = TableReference::from(relation);+ assert_eq!(recovered, TableReference::bare("has.dot"));+ }++ #[test]+ fn column_relation_round_trip_preserves_dotted_partial_reference() {+ let column = datafusion_common::Column::new(+ Some(TableReference::partial("my.schema", "table")),+ "column",+ );++ let proto: crate::protobuf_common::Column = (&column).into();+ let relation = proto.relation.expect("relation should be present");++ assert_eq!(relation.relation, "my.schema.table");+ assert_eq!(+ relation.parts,+ vec!["my.schema".to_string(), "table".to_string()]+ );++ let recovered = TableReference::from(relation);+ assert_eq!(recovered, TableReference::partial("my.schema", "table"));+ }++ #[test]+ fn column_relation_round_trip_preserves_dotted_full_reference() {+ let column = datafusion_common::Column::new(+ Some(TableReference::full("catalog", "my.schema", "table")),+ "column",+ );++ let proto: crate::protobuf_common::Column = (&column).into();+ let relation = proto.relation.expect("relation should be present");++ assert_eq!(relation.relation, "catalog.my.schema.table");+ assert_eq!(+ relation.parts,+ vec![+ "catalog".to_string(),+ "my.schema".to_string(),+ "table".to_string()+ ]+ );++ let recovered = TableReference::from(relation);+ assert_eq!(+ recovered,+ TableReference::full("catalog", "my.schema", "table")+ );+ }++ #[test]+ fn df_schema_round_trip_preserves_dotted_qualifiers() {+ use std::sync::Arc;++ use datafusion_common::DFSchema;+ use datafusion_common::arrow::datatypes::{DataType, Field};++ let schema = DFSchema::new_with_metadata(+ vec![+ (+ Some(TableReference::partial("my.schema", "table")),+ Arc::new(Field::new("partial_col", DataType::Utf8, true)),+ ),+ (+ Some(TableReference::full("catalog", "my.schema", "table")),+ Arc::new(Field::new("full_col", DataType::Utf8, true)),+ ),+ ],+ std::collections::HashMap::<String, String>::new(),+ )+ .unwrap();++ let proto: crate::protobuf_common::DfSchema = (&schema).try_into().unwrap();+ let recovered = DFSchema::try_from(&proto).unwrap();++ let qualifiers = recovered+ .iter()+ .map(|(qualifier, _)| qualifier.cloned())+ .collect::<Vec<_>>();++ assert_eq!(+ qualifiers,+ vec![+ Some(TableReference::partial("my.schema", "table")),+ Some(TableReference::full("catalog", "my.schema", "table")),+ ]+ );+ }++ #[test]+ fn column_relation_decodes_legacy_relation() {+ let proto = crate::protobuf_common::ColumnRelation {+ relation: "schema.table".to_string(),+ parts: vec![],+ };++ let recovered = TableReference::from(proto);++ assert_eq!(recovered, TableReference::partial("schema", "table"));+ }+#[test]fn table_parquet_options_defaults_missing_global() {let recovered = TableParquetOptions::try_from(diff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rsindex a48c7686bd..29145f4b28 100644--- a/datafusion/proto-common/src/generated/pbjson.rs+++ b/datafusion/proto-common/src/generated/pbjson.rs@@ -1030,10 +1030,16 @@ impl serde::Serialize for ColumnRelation {if !self.relation.is_empty() {len += 1;}+ if !self.parts.is_empty() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion_common.ColumnRelation", len)?;if !self.relation.is_empty() {struct_ser.serialize_field("relation", &self.relation)?;}+ if !self.parts.is_empty() {+ struct_ser.serialize_field("parts", &self.parts)?;+ }struct_ser.end()}}@@ -1045,11 +1051,13 @@ impl<'de> serde::Deserialize<'de> for ColumnRelation {{const FIELDS: &[&str] = &["relation",+ "parts",];#[allow(clippy::enum_variant_names)]enum GeneratedField {Relation,+ Parts,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -1072,6 +1080,7 @@ impl<'de> serde::Deserialize<'de> for ColumnRelation {{match value {"relation" => Ok(GeneratedField::Relation),+ "parts" => Ok(GeneratedField::Parts),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -1092,6 +1101,7 @@ impl<'de> serde::Deserialize<'de> for ColumnRelation {V: serde::de::MapAccess<'de>,{let mut relation__ = None;+ let mut parts__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Relation => {@@ -1100,10 +1110,17 @@ impl<'de> serde::Deserialize<'de> for ColumnRelation {}relation__ = Some(map_.next_value()?);}+ GeneratedField::Parts => {+ if parts__.is_some() {+ return Err(serde::de::Error::duplicate_field("parts"));+ }+ parts__ = Some(map_.next_value()?);+ }}}Ok(ColumnRelation {relation: relation__.unwrap_or_default(),+ parts: parts__.unwrap_or_default(),})}}diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rsindex 469e3434bc..7f85a65a76 100644--- a/datafusion/proto-common/src/generated/prost.rs+++ b/datafusion/proto-common/src/generated/prost.rs@@ -3,6 +3,8 @@pub struct ColumnRelation {#[prost(string, tag = "1")]pub relation: ::prost::alloc::string::String,+ #[prost(string, repeated, tag = "2")]+ pub parts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,}#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]pub struct Column {diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rsindex 1524fef026..6c51087941 100644--- a/datafusion/proto-common/src/to_proto/mod.rs+++ b/datafusion/proto-common/src/to_proto/mod.rs@@ -248,6 +248,7 @@ impl From<Column> for protobuf::Column {Self {relation: c.relation.map(|relation| protobuf::ColumnRelation {relation: relation.to_string(),+ parts: relation.to_vec(),}),name: c.name,}@@ -293,6 +294,7 @@ impl TryFrom<&DFSchema> for protobuf::DfSchema {field: Some(field.as_ref().try_into()?),qualifier: qualifier.map(|r| protobuf::ColumnRelation {relation: r.to_string(),+ parts: r.to_vec(),}),})})diff --git a/datafusion/proto-models/src/generated/datafusion_proto_common.rs b/datafusion/proto-models/src/generated/datafusion_proto_common.rsindex 469e3434bc..7f85a65a76 100644--- a/datafusion/proto-models/src/generated/datafusion_proto_common.rs+++ b/datafusion/proto-models/src/generated/datafusion_proto_common.rs@@ -3,6 +3,8 @@pub struct ColumnRelation {#[prost(string, tag = "1")]pub relation: ::prost::alloc::string::String,+ #[prost(string, repeated, tag = "2")]+ pub parts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,}#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]pub struct Column {