Skip to content

Apache DataFusion SQL Query Engine

fix: log metadata differences when comparing physical and logical schema (#19070)

Adam Curtisd4820d17194a
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
index 9e4c78a3d0..a942b5de41 100644
--- a/datafusion/core/src/physical_planner.rs
+++ b/datafusion/core/src/physical_planner.rs
@@ -687,6 +687,17 @@ impl DefaultPhysicalPlanner {
)
{
let mut differences = Vec::new();
+
+ if physical_input_schema.metadata()
+ != physical_input_schema_from_logical.metadata()
+ {
+ differences.push(format!(
+ "schema metadata differs: (physical) {:?} vs (logical) {:?}",
+ physical_input_schema.metadata(),
+ physical_input_schema_from_logical.metadata()
+ ));
+ }
+
if physical_input_schema.fields().len()
!= physical_input_schema_from_logical.fields().len()
{
@@ -716,6 +727,15 @@ impl DefaultPhysicalPlanner {
if physical_field.is_nullable() && !logical_field.is_nullable() {
differences.push(format!("field nullability at index {} [{}]: (physical) {} vs (logical) {}", i, physical_field.name(), physical_field.is_nullable(), logical_field.is_nullable()));
}
+ if physical_field.metadata() != logical_field.metadata() {
+ differences.push(format!(
+ "field metadata at index {} [{}]: (physical) {:?} vs (logical) {:?}",
+ i,
+ physical_field.name(),
+ physical_field.metadata(),
+ logical_field.metadata()
+ ));
+ }
}
return internal_err!("Physical input schema should be the same as the one converted from logical input schema. Differences: {}", differences
.iter()
@@ -3921,4 +3941,228 @@ digraph {
Ok(())
}
+
+ // --- Tests for aggregate schema mismatch error messages ---
+
+ use crate::catalog::TableProvider;
+ use datafusion_catalog::Session;
+ use datafusion_expr::TableType;
+
+ /// A TableProvider that returns schemas for logical planning vs physical planning.
+ /// Used to test schema mismatch error messages.
+ #[derive(Debug)]
+ struct MockSchemaTableProvider {
+ logical_schema: SchemaRef,
+ physical_schema: SchemaRef,
+ }
+
+ #[async_trait]
+ impl TableProvider for MockSchemaTableProvider {
+ fn as_any(&self) -> &dyn Any {
+ self
+ }
+
+ fn schema(&self) -> SchemaRef {
+ Arc::clone(&self.logical_schema)
+ }
+
+ fn table_type(&self) -> TableType {
+ TableType::Base
+ }
+
+ async fn scan(
+ &self,
+ _state: &dyn Session,
+ _projection: Option<&Vec<usize>>,
+ _filters: &[Expr],
+ _limit: Option<usize>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ Ok(Arc::new(NoOpExecutionPlan::new(Arc::clone(
+ &self.physical_schema,
+ ))))
+ }
+ }
+
+ /// Attempts to plan a query with potentially mismatched schemas.
+ async fn plan_with_schemas(
+ logical_schema: SchemaRef,
+ physical_schema: SchemaRef,
+ query: &str,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ let provider = MockSchemaTableProvider {
+ logical_schema,
+ physical_schema,
+ };
+ let ctx = SessionContext::new();
+ ctx.register_table("test", Arc::new(provider)).unwrap();
+
+ ctx.sql(query).await.unwrap().create_physical_plan().await
+ }
+
+ #[tokio::test]
+ // When schemas match, planning proceeds past the schema_satisfied_by check.
+ // It then panics on unimplemented error in NoOpExecutionPlan.
+ #[should_panic(expected = "NoOpExecutionPlan")]
+ async fn test_aggregate_schema_check_passes() {
+ let schema =
+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));
+
+ plan_with_schemas(
+ Arc::clone(&schema),
+ schema,
+ "SELECT count(*) FROM test GROUP BY c1",
+ )
+ .await
+ .unwrap();
+ }
+
+ #[tokio::test]
+ async fn test_aggregate_schema_mismatch_metadata() {
+ let logical_schema =
+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));
+ let physical_schema = Arc::new(
+ Schema::new(vec![Field::new("c1", DataType::Int32, false)])
+ .with_metadata(HashMap::from([("key".into(), "value".into())])),
+ );
+
+ let err = plan_with_schemas(
+ logical_schema,
+ physical_schema,
+ "SELECT count(*) FROM test GROUP BY c1",
+ )
+ .await
+ .unwrap_err();
+
+ assert_contains!(err.to_string(), "schema metadata differs");
+ }
+
+ #[tokio::test]
+ async fn test_aggregate_schema_mismatch_field_count() {
+ let logical_schema =
+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));
+ let physical_schema = Arc::new(Schema::new(vec![
+ Field::new("c1", DataType::Int32, false),
+ Field::new("c2", DataType::Int32, false),
+ ]));
+
+ let err = plan_with_schemas(
+ logical_schema,
+ physical_schema,
+ "SELECT count(*) FROM test GROUP BY c1",
+ )
+ .await
+ .unwrap_err();
+
+ assert_contains!(err.to_string(), "Different number of fields");
+ }
+
+ #[tokio::test]
+ async fn test_aggregate_schema_mismatch_field_name() {
+ let logical_schema =
+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));
+ let physical_schema = Arc::new(Schema::new(vec![Field::new(
+ "different_name",
+ DataType::Int32,
+ false,
+ )]));
+
+ let err = plan_with_schemas(
+ logical_schema,
+ physical_schema,
+ "SELECT count(*) FROM test GROUP BY c1",
+ )
+ .await
+ .unwrap_err();
+
+ assert_contains!(err.to_string(), "field name at index");
+ }
+
+ #[tokio::test]
+ async fn test_aggregate_schema_mismatch_field_type() {
+ let logical_schema =
+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));
+ let physical_schema =
+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int64, false)]));
+
+ let err = plan_with_schemas(
+ logical_schema,
+ physical_schema,
+ "SELECT count(*) FROM test GROUP BY c1",
+ )
+ .await
+ .unwrap_err();
+
+ assert_contains!(err.to_string(), "field data type at index");
+ }
+
+ #[tokio::test]
+ async fn test_aggregate_schema_mismatch_field_nullability() {
+ let logical_schema =
+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));
+ let physical_schema =
+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, true)]));
+
+ let err = plan_with_schemas(
+ logical_schema,
+ physical_schema,
+ "SELECT count(*) FROM test GROUP BY c1",
+ )
+ .await
+ .unwrap_err();
+
+ assert_contains!(err.to_string(), "field nullability at index");
+ }
+
+ #[tokio::test]
+ async fn test_aggregate_schema_mismatch_field_metadata() {
+ let logical_schema =
+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));
+ let physical_schema =
+ Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)
+ .with_metadata(HashMap::from([("key".into(), "value".into())]))]));
+
+ let err = plan_with_schemas(
+ logical_schema,
+ physical_schema,
+ "SELECT count(*) FROM test GROUP BY c1",
+ )
+ .await
+ .unwrap_err();
+
+ assert_contains!(err.to_string(), "field metadata at index");
+ }
+
+ #[tokio::test]
+ async fn test_aggregate_schema_mismatch_multiple() {
+ let logical_schema = Arc::new(Schema::new(vec![
+ Field::new("c1", DataType::Int32, false),
+ Field::new("c2", DataType::Utf8, false),
+ ]));
+ let physical_schema = Arc::new(
+ Schema::new(vec![
+ Field::new("c1", DataType::Int64, true)
+ .with_metadata(HashMap::from([("key".into(), "value".into())])),
+ Field::new("c2", DataType::Utf8, false),
+ ])
+ .with_metadata(HashMap::from([(
+ "schema_key".into(),
+ "schema_value".into(),
+ )])),
+ );
+
+ let err = plan_with_schemas(
+ logical_schema,
+ physical_schema,
+ "SELECT count(*) FROM test GROUP BY c1",
+ )
+ .await
+ .unwrap_err();
+
+ // Verify all applicable error fragments are present
+ let err_str = err.to_string();
+ assert_contains!(&err_str, "schema metadata differs");
+ assert_contains!(&err_str, "field data type at index");
+ assert_contains!(&err_str, "field nullability at index");
+ assert_contains!(&err_str, "field metadata at index");
+ }
}