Skip to content

Apache DataFusion SQL Query Engine

fix: preserve total_byte_size in calculate_total_byte_size when num_r… (#24027)

Bert Vermeiren09dd8d246539
diff --git a/datafusion/common/src/stats.rs b/datafusion/common/src/stats.rs
index b7db556ee8..1a226c3698 100644
--- a/datafusion/common/src/stats.rs
+++ b/datafusion/common/src/stats.rs
@@ -427,7 +427,9 @@ impl Statistics {
}
/// Calculates `total_byte_size` based on the schema and `num_rows`.
- /// If any of the columns has non-primitive width, `total_byte_size` is set to inexact.
+ /// If any of the columns has non-primitive width, or `num_rows` is unknown,
+ /// the previous `total_byte_size` is kept but downgraded to inexact rather
+ /// than discarded.
pub fn calculate_total_byte_size(&mut self, schema: &Schema) {
let mut row_size = Some(0);
for field in schema.fields() {
@@ -441,11 +443,11 @@ impl Statistics {
}
}
}
- match row_size {
- None => {
+ match (row_size, &self.num_rows) {
+ (None, _) | (Some(_), Precision::Absent) => {
self.total_byte_size = self.total_byte_size.to_inexact();
}
- Some(size) => {
+ (Some(size), _) => {
self.total_byte_size = self.num_rows.multiply(&Precision::Exact(size));
}
}
@@ -3345,4 +3347,33 @@ mod tests {
precision_add_for_sum_in_place(&mut lhs, &Precision::Absent);
assert_eq!(lhs, Precision::Absent);
}
+
+ #[test]
+ fn test_calculate_total_byte_size() {
+ let primitive_schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+ let non_primitive_schema =
+ Schema::new(vec![Field::new("a", DataType::Utf8, false)]);
+
+ // All-primitive schema with a known row count computes an exact size.
+ let mut stats = Statistics::new_unknown(&primitive_schema);
+ stats.num_rows = Precision::Exact(10);
+ stats.calculate_total_byte_size(&primitive_schema);
+ assert_eq!(stats.total_byte_size, Precision::Exact(40));
+
+ // All-primitive schema with an unknown row count keeps a previously
+ // known `total_byte_size`, downgraded to inexact, instead of
+ // discarding it to `Absent`.
+ let mut stats = Statistics::new_unknown(&primitive_schema);
+ stats.total_byte_size = Precision::Exact(1234);
+ stats.calculate_total_byte_size(&primitive_schema);
+ assert_eq!(stats.total_byte_size, Precision::Inexact(1234));
+
+ // Non-primitive schema always downgrades any existing
+ // `total_byte_size` to inexact, regardless of `num_rows`.
+ let mut stats = Statistics::new_unknown(&non_primitive_schema);
+ stats.num_rows = Precision::Exact(10);
+ stats.total_byte_size = Precision::Exact(999);
+ stats.calculate_total_byte_size(&non_primitive_schema);
+ assert_eq!(stats.total_byte_size, Precision::Inexact(999));
+ }
}