Skip to content

Apache DataFusion SQL Query Engine

feat: derive a distinct count from primary key and unique constraints (#24520)

Daniël Heres3931ab4471dc
diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs
index 91dcd5b76f..73f8b8afa1 100644
--- a/datafusion/datasource/src/file_scan_config/mod.rs
+++ b/datafusion/datasource/src/file_scan_config/mod.rs
@@ -39,7 +39,8 @@ use arrow::datatypes::{DataType, Schema, SchemaRef};
use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
- Constraints, Result, ScalarValue, Statistics, internal_datafusion_err, internal_err,
+ Constraint, Constraints, Result, ScalarValue, Statistics, internal_datafusion_err,
+ internal_err,
};
use datafusion_execution::{
SendableRecordBatchStream, TaskContext, object_store::ObjectStoreUrl,
@@ -542,9 +543,10 @@ impl FileScanConfigBuilder {
} = self;
let constraints = constraints.unwrap_or_default();
- let statistics = statistics.unwrap_or_else(|| {
+ let mut statistics = statistics.unwrap_or_else(|| {
Statistics::new_unknown(file_source.table_schema().table_schema())
});
+ add_key_distinct_counts(&constraints, &mut statistics);
let file_compression_type =
file_compression_type.unwrap_or(FileCompressionType::UNCOMPRESSED);
@@ -568,6 +570,32 @@ impl FileScanConfigBuilder {
}
}
+/// Records that a key column holds one distinct value per row, which no file format
+/// stores. Single-column keys only: a composite key says nothing about its columns.
+fn add_key_distinct_counts(constraints: &Constraints, statistics: &mut Statistics) {
+ let num_rows = statistics.num_rows;
+ for constraint in constraints.iter() {
+ let (Constraint::PrimaryKey(indices) | Constraint::Unique(indices)) = constraint;
+ let [index] = indices[..] else {
+ continue;
+ };
+ let Some(column) = statistics.column_statistics.get_mut(index) else {
+ continue;
+ };
+ if column.distinct_count != Precision::Absent {
+ continue;
+ }
+ // A NULL is not a distinct value. A primary key has none; a unique column may
+ // repeat them, so an unknown count leaves the result inexact.
+ let nulls = match (constraint, column.null_count) {
+ (Constraint::PrimaryKey(_), Precision::Absent) => Precision::Exact(0),
+ (_, Precision::Absent) => Precision::Inexact(0),
+ (_, nulls) => nulls,
+ };
+ column.distinct_count = num_rows.sub(&nulls);
+ }
+}
+
impl From<FileScanConfig> for FileScanConfigBuilder {
fn from(config: FileScanConfig) -> Self {
Self {
@@ -2133,6 +2161,117 @@ mod tests {
}
// sets default for configs that play no role in projections
+ fn config_with_constraints(
+ table_schema: TableSchema,
+ statistics: Statistics,
+ constraints: Vec<Constraint>,
+ projection: Option<Vec<usize>>,
+ ) -> FileScanConfig {
+ FileScanConfigBuilder::new(
+ ObjectStoreUrl::parse("test:///").unwrap(),
+ Arc::new(MockSource::new(table_schema)),
+ )
+ .with_statistics(statistics)
+ .with_constraints(Constraints::new_unverified(constraints))
+ .with_projection_indices(projection)
+ .unwrap()
+ .build()
+ }
+
+ /// A key column has one distinct value per row, which no file format records.
+ #[test]
+ fn key_columns_report_a_distinct_count() {
+ let file_schema = Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new("part", DataType::Int32, false),
+ Field::new("code", DataType::Int32, true),
+ ]));
+ let table_schema = TableSchema::builder(Arc::clone(&file_schema)).build();
+ let mut statistics = Statistics::new_unknown(&file_schema);
+ statistics.num_rows = Precision::Exact(100);
+ // A unique column may repeat NULL, which is not a distinct value.
+ statistics.column_statistics[2].null_count = Precision::Exact(10);
+
+ let stats = |constraints| {
+ config_with_constraints(
+ table_schema.clone(),
+ statistics.clone(),
+ constraints,
+ None,
+ )
+ .statistics()
+ };
+
+ // A primary key cannot be null, so the count is as exact as the row count.
+ let primary_key = stats(vec![Constraint::PrimaryKey(vec![0])]);
+ assert_eq!(
+ primary_key.column_statistics[0].distinct_count,
+ Precision::Exact(100)
+ );
+ assert_eq!(
+ primary_key.column_statistics[1].distinct_count,
+ Precision::Absent
+ );
+
+ // The nulls a unique column may repeat are known here, so this is exact too.
+ let unique = stats(vec![Constraint::Unique(vec![2])]);
+ assert_eq!(
+ unique.column_statistics[2].distinct_count,
+ Precision::Exact(90)
+ );
+
+ // With an unknown null count it is not exact.
+ let mut unknown_nulls = statistics.clone();
+ unknown_nulls.column_statistics[2].null_count = Precision::Absent;
+ let unique = config_with_constraints(
+ table_schema.clone(),
+ unknown_nulls,
+ vec![Constraint::Unique(vec![2])],
+ None,
+ )
+ .statistics();
+ assert_eq!(
+ unique.column_statistics[2].distinct_count,
+ Precision::Inexact(100)
+ );
+
+ // A composite key leaves its columns alone: only the combination is unique.
+ let composite = stats(vec![Constraint::PrimaryKey(vec![0, 1])]);
+ assert_eq!(
+ composite.column_statistics[0].distinct_count,
+ Precision::Absent
+ );
+ assert_eq!(
+ composite.column_statistics[1].distinct_count,
+ Precision::Absent
+ );
+ }
+
+ /// The count has to reach the plan, which reads statistics through the projection.
+ #[test]
+ fn a_projected_scan_keeps_the_key_distinct_count() {
+ let file_schema = Arc::new(Schema::new(vec![
+ Field::new("value", DataType::Int32, true),
+ Field::new("id", DataType::Int32, false),
+ ]));
+ let table_schema = TableSchema::builder(Arc::clone(&file_schema)).build();
+ let mut statistics = Statistics::new_unknown(&file_schema);
+ statistics.num_rows = Precision::Exact(100);
+
+ let config = config_with_constraints(
+ table_schema,
+ statistics,
+ vec![Constraint::PrimaryKey(vec![1])],
+ Some(vec![1]),
+ );
+
+ let projected = config.partition_statistics(None).unwrap();
+ assert_eq!(
+ projected.column_statistics[0].distinct_count,
+ Precision::Exact(100)
+ );
+ }
+
fn config_for_projection(
file_schema: SchemaRef,
projection: Option<Vec<usize>>,
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index 186c49562b..7cf14ac57d 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -63,7 +63,7 @@ use datafusion_execution::TaskContext;
use datafusion_expr::Operator;
use datafusion_physical_expr::equivalence::ProjectionMapping;
use datafusion_physical_expr::expressions::{
- BinaryExpr, Column, IsNotNullExpr, Literal, lit,
+ BinaryExpr, Column, InListExpr, IsNotNullExpr, Literal, lit,
};
use datafusion_physical_expr::intervals::utils::check_support;
use datafusion_physical_expr::utils::{collect_columns, reassign_expr_columns};
@@ -343,6 +343,11 @@ impl FilterExec {
let input_num_rows = input_stats.num_rows;
let input_total_byte_size = input_stats.total_byte_size;
+ // A column holding each of its values once, as a primary key or unique
+ // constraint says, matches one row per value asked for. No selectivity
+ // expresses that.
+ let match_limit = unique_match_limit(predicate, &input_stats);
+
let (selectivity, num_rows, column_statistics) = if is_infeasible {
// Contradictory predicate: no rows survive. Row-bounded counts are
// zero; value statistics are undefined on an empty column.
@@ -406,6 +411,10 @@ impl FilterExec {
}
};
+ let num_rows = match (match_limit, num_rows.get_value()) {
+ (Some(limit), Some(rows)) if *rows > limit => Precision::Inexact(limit),
+ _ => num_rows,
+ };
let total_byte_size =
scale_byte_size_at_rows(input_total_byte_size, selectivity, num_rows);
@@ -972,6 +981,76 @@ impl EmbeddedProjection for FilterExec {
}
}
+/// The most rows a filter can match, when it restricts a column holding each value
+/// once to a fixed set of values: one row per value.
+fn unique_match_limit(
+ predicate: &Arc<dyn PhysicalExpr>,
+ statistics: &Statistics,
+) -> Option<usize> {
+ let mut limit: Option<usize> = None;
+ for expr in split_conjunction(predicate) {
+ let Some((index, values)) = restricted_column(expr) else {
+ continue;
+ };
+ let holds_once = statistics
+ .column_statistics
+ .get(index)
+ .is_some_and(|column| holds_each_value_once(column, &statistics.num_rows));
+ if !holds_once {
+ continue;
+ }
+ limit = Some(limit.map_or(values, |limit: usize| limit.min(values)));
+ }
+ limit
+}
+
+/// The column an expression restricts to a fixed set of values, and how many values
+/// that is. NULL is never one of them: it matches nothing.
+fn restricted_column(expr: &Arc<dyn PhysicalExpr>) -> Option<(usize, usize)> {
+ if let Some(in_list) = expr.downcast_ref::<InListExpr>() {
+ if in_list.negated() {
+ return None;
+ }
+ let column = in_list.expr().downcast_ref::<Column>()?;
+ let mut values: Vec<&ScalarValue> = vec![];
+ for expr in in_list.list() {
+ let value = expr.downcast_ref::<Literal>()?.value();
+ if !value.is_null() && !values.contains(&value) {
+ values.push(value);
+ }
+ }
+ return Some((column.index(), values.len()));
+ }
+
+ let binary = expr.downcast_ref::<BinaryExpr>()?;
+ if *binary.op() != Operator::Eq {
+ return None;
+ }
+ let (column, literal) = match (
+ binary.left().downcast_ref::<Column>(),
+ binary.right().downcast_ref::<Column>(),
+ ) {
+ (Some(column), None) => (column, binary.right()),
+ (None, Some(column)) => (column, binary.left()),
+ _ => return None,
+ };
+ let value = literal.downcast_ref::<Literal>()?.value();
+ (!value.is_null()).then_some((column.index(), 1))
+}
+
+/// Whether the column has as many distinct values as it has non-null rows, so each
+/// value appears once.
+fn holds_each_value_once(column: &ColumnStatistics, num_rows: &Precision<usize>) -> bool {
+ let (Some(rows), Some(distinct), Some(nulls)) = (
+ num_rows.get_value(),
+ column.distinct_count.get_value(),
+ column.null_count.get_value(),
+ ) else {
+ return false;
+ };
+ distinct.saturating_add(*nulls) >= *rows
+}
+
/// Collects column equality information from `col = literal` predicates in a
/// conjunction.
///
@@ -1486,6 +1565,100 @@ mod tests {
Ok(())
}
+ /// An equality on a column that holds each value once matches one row at most,
+ /// including on a type interval analysis cannot read, where the default
+ /// selectivity would otherwise apply.
+ #[tokio::test]
+ async fn test_filter_statistics_equality_on_a_unique_column() -> Result<()> {
+ let schema = Schema::new(vec![Field::new("id", DataType::Utf8, true)]);
+ let unique = ColumnStatistics {
+ null_count: Precision::Exact(0),
+ distinct_count: Precision::Exact(100),
+ ..Default::default()
+ };
+ let rows = |column: ColumnStatistics| -> Result<Precision<usize>> {
+ let input = Arc::new(StatisticsExec::new(
+ Statistics {
+ num_rows: Precision::Exact(100),
+ total_byte_size: Precision::Exact(800),
+ column_statistics: vec![column],
+ },
+ schema.clone(),
+ ));
+ let predicate =
+ binary(col("id", &schema)?, Operator::Eq, lit("seven"), &schema)?;
+ let filter: Arc<dyn ExecutionPlan> =
+ Arc::new(FilterExec::try_new(predicate, input)?);
+ Ok(StatisticsContext::new()
+ .compute(filter.as_ref(), &StatisticsArgs::new())?
+ .num_rows)
+ };
+
+ assert_eq!(rows(unique.clone())?, Precision::Inexact(1));
+
+ // The nulls a unique column may repeat do not make it hold a value twice.
+ assert_eq!(
+ rows(ColumnStatistics {
+ null_count: Precision::Exact(10),
+ distinct_count: Precision::Exact(90),
+ ..unique.clone()
+ })?,
+ Precision::Inexact(1)
+ );
+
+ // Without a distinct count, the default selectivity applies as before.
+ assert_eq!(
+ rows(ColumnStatistics {
+ distinct_count: Precision::Absent,
+ ..unique
+ })?,
+ Precision::Inexact(20)
+ );
+
+ Ok(())
+ }
+
+ /// Asking a unique column for three values matches three rows at most.
+ #[tokio::test]
+ async fn test_filter_statistics_in_list_on_a_unique_column() -> Result<()> {
+ use datafusion_physical_expr::expressions::in_list;
+
+ let schema = Schema::new(vec![Field::new("id", DataType::Utf8, true)]);
+ let rows = |list: Vec<&str>, negated: bool| -> Result<Precision<usize>> {
+ let input = Arc::new(StatisticsExec::new(
+ Statistics {
+ num_rows: Precision::Exact(100),
+ total_byte_size: Precision::Exact(800),
+ column_statistics: vec![ColumnStatistics {
+ null_count: Precision::Exact(0),
+ distinct_count: Precision::Exact(100),
+ ..Default::default()
+ }],
+ },
+ schema.clone(),
+ ));
+ let predicate = in_list(
+ col("id", &schema)?,
+ list.into_iter().map(|value| lit(value) as _).collect(),
+ &negated,
+ &schema,
+ )?;
+ let filter: Arc<dyn ExecutionPlan> =
+ Arc::new(FilterExec::try_new(predicate, input)?);
+ Ok(StatisticsContext::new()
+ .compute(filter.as_ref(), &StatisticsArgs::new())?
+ .num_rows)
+ };
+
+ assert_eq!(rows(vec!["a", "b", "c"], false)?, Precision::Inexact(3));
+ // Repeats ask for the same row twice.
+ assert_eq!(rows(vec!["a", "b", "a"], false)?, Precision::Inexact(2));
+ // `NOT IN` selects nearly everything, so the default applies.
+ assert_eq!(rows(vec!["a", "b", "c"], true)?, Precision::Inexact(20));
+
+ Ok(())
+ }
+
#[tokio::test]
async fn test_filter_statistics_basic_expr() -> Result<()> {
// Table: