Apache DataFusion SQL Query Engine
fix: log metadata differences when comparing physical and logical schema (#19070)
Adam Curtisd4820d17194a
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 9e4c78a3d0..a942b5de41 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -687,6 +687,17 @@ impl DefaultPhysicalPlanner {){let mut differences = Vec::new();++ if physical_input_schema.metadata()+ != physical_input_schema_from_logical.metadata()+ {+ differences.push(format!(+ "schema metadata differs: (physical) {:?} vs (logical) {:?}",+ physical_input_schema.metadata(),+ physical_input_schema_from_logical.metadata()+ ));+ }+if physical_input_schema.fields().len()!= physical_input_schema_from_logical.fields().len(){@@ -716,6 +727,15 @@ impl DefaultPhysicalPlanner {if physical_field.is_nullable() && !logical_field.is_nullable() {differences.push(format!("field nullability at index {} [{}]: (physical) {} vs (logical) {}", i, physical_field.name(), physical_field.is_nullable(), logical_field.is_nullable()));}+ if physical_field.metadata() != logical_field.metadata() {+ differences.push(format!(+ "field metadata at index {} [{}]: (physical) {:?} vs (logical) {:?}",+ i,+ physical_field.name(),+ physical_field.metadata(),+ logical_field.metadata()+ ));+ }}return internal_err!("Physical input schema should be the same as the one converted from logical input schema. Differences: {}", differences.iter()@@ -3921,4 +3941,228 @@ digraph {Ok(())}++ // --- Tests for aggregate schema mismatch error messages ---++ use crate::catalog::TableProvider;+ use datafusion_catalog::Session;+ use datafusion_expr::TableType;++ /// A TableProvider that returns schemas for logical planning vs physical planning.+ /// Used to test schema mismatch error messages.+ #[derive(Debug)]+ struct MockSchemaTableProvider {+ logical_schema: SchemaRef,+ physical_schema: SchemaRef,+ }++ #[async_trait]+ impl TableProvider for MockSchemaTableProvider {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn schema(&self) -> SchemaRef {+ Arc::clone(&self.logical_schema)+ }++ fn table_type(&self) -> TableType {+ TableType::Base+ }++ async fn scan(+ &self,+ _state: &dyn Session,+ _projection: Option<&Vec<usize>>,+ _filters: &[Expr],+ _limit: Option<usize>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(Arc::new(NoOpExecutionPlan::new(Arc::clone(+ &self.physical_schema,+ ))))+ }+ }++ /// Attempts to plan a query with potentially mismatched schemas.+ async fn plan_with_schemas(+ logical_schema: SchemaRef,+ physical_schema: SchemaRef,+ query: &str,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let provider = MockSchemaTableProvider {+ logical_schema,+ physical_schema,+ };+ let ctx = SessionContext::new();+ ctx.register_table("test", Arc::new(provider)).unwrap();++ ctx.sql(query).await.unwrap().create_physical_plan().await+ }++ #[tokio::test]+ // When schemas match, planning proceeds past the schema_satisfied_by check.+ // It then panics on unimplemented error in NoOpExecutionPlan.+ #[should_panic(expected = "NoOpExecutionPlan")]+ async fn test_aggregate_schema_check_passes() {+ let schema =+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));++ plan_with_schemas(+ Arc::clone(&schema),+ schema,+ "SELECT count(*) FROM test GROUP BY c1",+ )+ .await+ .unwrap();+ }++ #[tokio::test]+ async fn test_aggregate_schema_mismatch_metadata() {+ let logical_schema =+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));+ let physical_schema = Arc::new(+ Schema::new(vec![Field::new("c1", DataType::Int32, false)])+ .with_metadata(HashMap::from([("key".into(), "value".into())])),+ );++ let err = plan_with_schemas(+ logical_schema,+ physical_schema,+ "SELECT count(*) FROM test GROUP BY c1",+ )+ .await+ .unwrap_err();++ assert_contains!(err.to_string(), "schema metadata differs");+ }++ #[tokio::test]+ async fn test_aggregate_schema_mismatch_field_count() {+ let logical_schema =+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));+ let physical_schema = Arc::new(Schema::new(vec![+ Field::new("c1", DataType::Int32, false),+ Field::new("c2", DataType::Int32, false),+ ]));++ let err = plan_with_schemas(+ logical_schema,+ physical_schema,+ "SELECT count(*) FROM test GROUP BY c1",+ )+ .await+ .unwrap_err();++ assert_contains!(err.to_string(), "Different number of fields");+ }++ #[tokio::test]+ async fn test_aggregate_schema_mismatch_field_name() {+ let logical_schema =+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));+ let physical_schema = Arc::new(Schema::new(vec![Field::new(+ "different_name",+ DataType::Int32,+ false,+ )]));++ let err = plan_with_schemas(+ logical_schema,+ physical_schema,+ "SELECT count(*) FROM test GROUP BY c1",+ )+ .await+ .unwrap_err();++ assert_contains!(err.to_string(), "field name at index");+ }++ #[tokio::test]+ async fn test_aggregate_schema_mismatch_field_type() {+ let logical_schema =+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));+ let physical_schema =+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int64, false)]));++ let err = plan_with_schemas(+ logical_schema,+ physical_schema,+ "SELECT count(*) FROM test GROUP BY c1",+ )+ .await+ .unwrap_err();++ assert_contains!(err.to_string(), "field data type at index");+ }++ #[tokio::test]+ async fn test_aggregate_schema_mismatch_field_nullability() {+ let logical_schema =+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));+ let physical_schema =+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, true)]));++ let err = plan_with_schemas(+ logical_schema,+ physical_schema,+ "SELECT count(*) FROM test GROUP BY c1",+ )+ .await+ .unwrap_err();++ assert_contains!(err.to_string(), "field nullability at index");+ }++ #[tokio::test]+ async fn test_aggregate_schema_mismatch_field_metadata() {+ let logical_schema =+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));+ let physical_schema =+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)+ .with_metadata(HashMap::from([("key".into(), "value".into())]))]));++ let err = plan_with_schemas(+ logical_schema,+ physical_schema,+ "SELECT count(*) FROM test GROUP BY c1",+ )+ .await+ .unwrap_err();++ assert_contains!(err.to_string(), "field metadata at index");+ }++ #[tokio::test]+ async fn test_aggregate_schema_mismatch_multiple() {+ let logical_schema = Arc::new(Schema::new(vec![+ Field::new("c1", DataType::Int32, false),+ Field::new("c2", DataType::Utf8, false),+ ]));+ let physical_schema = Arc::new(+ Schema::new(vec![+ Field::new("c1", DataType::Int64, true)+ .with_metadata(HashMap::from([("key".into(), "value".into())])),+ Field::new("c2", DataType::Utf8, false),+ ])+ .with_metadata(HashMap::from([(+ "schema_key".into(),+ "schema_value".into(),+ )])),+ );++ let err = plan_with_schemas(+ logical_schema,+ physical_schema,+ "SELECT count(*) FROM test GROUP BY c1",+ )+ .await+ .unwrap_err();++ // Verify all applicable error fragments are present+ let err_str = err.to_string();+ assert_contains!(&err_str, "schema metadata differs");+ assert_contains!(&err_str, "field data type at index");+ assert_contains!(&err_str, "field nullability at index");+ assert_contains!(&err_str, "field metadata at index");+ }}