Apache DataFusion SQL Query Engine
Fix: Do not normalize table names when deserializing from protobuf (#18187)
Aldrin M167baf718278
diff --git a/datafusion-cli/src/object_storage/instrumented.rs b/datafusion-cli/src/object_storage/instrumented.rsindex 4465c59a90..c4b63b417f 100644--- a/datafusion-cli/src/object_storage/instrumented.rs+++ b/datafusion-cli/src/object_storage/instrumented.rs@@ -58,7 +58,7 @@ pub enum InstrumentedObjectStoreMode {impl fmt::Display for InstrumentedObjectStoreMode {fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- write!(f, "{:?}", self)+ write!(f, "{self:?}")}}@@ -426,7 +426,7 @@ pub enum Operation {impl fmt::Display for Operation {fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- write!(f, "{:?}", self)+ write!(f, "{self:?}")}}@@ -556,11 +556,11 @@ impl RequestSummaries {let size_stats = s.size_stats.as_ref();let dur_avg = duration_stats.map(|d| {let avg = d.sum.as_secs_f32() / count;- format!("{:.6}s", avg)+ format!("{avg:.6}s")});let size_avg = size_stats.map(|s| {let avg = s.sum as f32 / count;- format!("{} B", avg)+ format!("{avg} B")});[dur_avg, size_avg]})diff --git a/datafusion-cli/src/print_options.rs b/datafusion-cli/src/print_options.rsindex 5804617f39..93d1d450fd 100644--- a/datafusion-cli/src/print_options.rs+++ b/datafusion-cli/src/print_options.rs@@ -206,7 +206,7 @@ impl PrintOptions {writeln!(writer, "Summaries:")?;let summaries = RequestSummaries::new(&requests);- writeln!(writer, "{}", summaries)?;+ writeln!(writer, "{summaries}")?;}}}diff --git a/datafusion/common/src/table_reference.rs b/datafusion/common/src/table_reference.rsindex 7cf8e7af1a..5744658567 100644--- a/datafusion/common/src/table_reference.rs+++ b/datafusion/common/src/table_reference.rs@@ -269,24 +269,41 @@ impl TableReference {}/// Forms a [`TableReference`] by parsing `s` as a multipart SQL- /// identifier. See docs on [`TableReference`] for more details.+ /// identifier, normalizing `s` to lowercase.+ /// See docs on [`TableReference`] for more details.pub fn parse_str(s: &str) -> Self {- let mut parts = parse_identifiers_normalized(s, false);+ Self::parse_str_normalized(s, false)+ }++ /// Forms a [`TableReference`] by parsing `s` as a multipart SQL+ /// identifier, normalizing `s` to lowercase if `ignore_case` is `false`.+ /// See docs on [`TableReference`] for more details.+ pub fn parse_str_normalized(s: &str, ignore_case: bool) -> Self {+ let table_parts = parse_identifiers_normalized(s, ignore_case);+ Self::from_vec(table_parts).unwrap_or_else(|| Self::Bare { table: s.into() })+ }++ /// Consume a vector of identifier parts to compose a [`TableReference`]. The input vector+ /// should contain 1 <= N <= 3 elements in the following sequence:+ /// ```no_rust+ /// [<catalog>, <schema>, table]+ /// ```+ fn from_vec(mut parts: Vec<String>) -> Option<Self> {match parts.len() {- 1 => Self::Bare {- table: parts.remove(0).into(),- },- 2 => Self::Partial {- schema: parts.remove(0).into(),- table: parts.remove(0).into(),- },- 3 => Self::Full {- catalog: parts.remove(0).into(),- schema: parts.remove(0).into(),- table: parts.remove(0).into(),- },- _ => Self::Bare { table: s.into() },+ 1 => Some(Self::Bare {+ table: parts.pop()?.into(),+ }),+ 2 => Some(Self::Partial {+ table: parts.pop()?.into(),+ schema: parts.pop()?.into(),+ }),+ 3 => Some(Self::Full {+ table: parts.pop()?.into(),+ schema: parts.pop()?.into(),+ catalog: parts.pop()?.into(),+ }),+ _ => None,}}diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rsindex c72e3b3a8d..045c02a5a2 100644--- a/datafusion/common/src/utils/mod.rs+++ b/datafusion/common/src/utils/mod.rs@@ -285,6 +285,9 @@ pub(crate) fn parse_identifiers(s: &str) -> Result<Vec<Ident>> {Ok(idents)}+/// Parse a string into a vector of identifiers.+///+/// Note: If ignore_case is false, the string will be normalized to lowercase.#[cfg(feature = "sql")]pub(crate) fn parse_identifiers_normalized(s: &str, ignore_case: bool) -> Vec<String> {parse_identifiers(s)diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex 2d07fb8410..4ede5b970e 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -138,11 +138,17 @@ where}}+impl From<protobuf::ColumnRelation> for TableReference {+ fn from(rel: protobuf::ColumnRelation) -> Self {+ Self::parse_str_normalized(rel.relation.as_str(), true)+ }+}+impl From<protobuf::Column> for Column {fn from(c: protobuf::Column) -> Self {let protobuf::Column { relation, name } = c;- Self::new(relation.map(|r| r.relation), name)+ Self::new(relation, name)}}@@ -164,10 +170,7 @@ impl TryFrom<&protobuf::DfSchema> for DFSchema {.map(|df_field| {let field: Field = df_field.field.as_ref().required("field")?;Ok((- df_field- .qualifier- .as_ref()- .map(|q| q.relation.clone().into()),+ df_field.qualifier.as_ref().map(|q| q.clone().into()),Arc::new(field),))})diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rsindex 4296867049..507a0cec9d 100644--- a/datafusion/proto/src/logical_plan/from_proto.rs+++ b/datafusion/proto/src/logical_plan/from_proto.rs@@ -315,8 +315,7 @@ pub fn parse_expr(let null_treatment = protobuf::NullTreatment::try_from(null_treatment).map_err(|_| {proto_error(format!(- "Received a WindowExprNode message with unknown NullTreatment {}",- null_treatment+ "Received a WindowExprNode message with unknown NullTreatment {null_treatment}",))})?;Some(NullTreatment::from(null_treatment))@@ -596,8 +595,7 @@ pub fn parse_expr(let null_treatment = protobuf::NullTreatment::try_from(null_treatment).map_err(|_| {proto_error(format!(- "Received an AggregateUdfExprNode message with unknown NullTreatment {}",- null_treatment+ "Received an AggregateUdfExprNode message with unknown NullTreatment {null_treatment}",))})?;Some(NullTreatment::from(null_treatment))diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex 516f178cc0..18cd8b8e66 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -2804,3 +2804,49 @@ async fn roundtrip_arrow_scan() -> Result<()> {assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));Ok(())}++#[tokio::test]+async fn roundtrip_mixed_case_table_reference() -> Result<()> {+ // Prepare "client" database+ let client_ctx = SessionContext::new_with_config(+ SessionConfig::new()+ .set_bool("datafusion.sql_parser.enable_ident_normalization", false),+ );+ client_ctx+ .register_csv(+ "\"TestData\"",+ "tests/testdata/test.csv",+ CsvReadOptions::default(),+ )+ .await?;++ // Prepare "server" database+ let server_ctx = SessionContext::new_with_config(+ SessionConfig::new()+ .set_bool("datafusion.sql_parser.enable_ident_normalization", false),+ );+ server_ctx+ .register_csv(+ "\"TestData\"",+ "tests/testdata/test.csv",+ CsvReadOptions::default(),+ )+ .await?;++ // Create a logical plan, serialize it (client), then deserialize it (server)+ let dataframe = client_ctx+ .sql("SELECT a FROM TestData WHERE TestData.a = 1")+ .await?;++ let client_logical_plan = dataframe.into_optimized_plan()?;+ let plan_bytes = logical_plan_to_bytes(&client_logical_plan)?;+ let server_logical_plan =+ logical_plan_from_bytes(&plan_bytes, &server_ctx.task_ctx())?;++ assert_eq!(+ format!("{}", client_logical_plan.display_indent_schema()),+ format!("{}", server_logical_plan.display_indent_schema())+ );++ Ok(())+}