Apache DataFusion SQL Query Engine
deprecate: mark Statistics V2 framework (PR #14699) as deprecated (#22071)
Andrew Lambfe1dd577f706
diff --git a/datafusion/expr-common/src/statistics.rs b/datafusion/expr-common/src/statistics.rsindex 6c8cef35b3..c94c181615 100644--- a/datafusion/expr-common/src/statistics.rs+++ b/datafusion/expr-common/src/statistics.rs@@ -15,6 +15,16 @@// specific language governing permissions and limitations// under the License.+//! Probabilistic distributions for expression-level statistics (unused).+//!+//! Note: All public items in this module are **deprecated** as of `54.0.0`.+//!+//! See <https://github.com/apache/datafusion/pull/22071> for details.++// The whole module is deprecated; suppress warnings from intra-module uses+// of the deprecated types so the module continues to compile.+#![allow(deprecated)]+use std::f64::consts::LN_2;use crate::interval_arithmetic::{Interval, apply_operator};@@ -37,6 +47,10 @@ use datafusion_common::{/// is the main unit of calculus when evaluating expressions in a statistical/// context. Notions like column and table statistics are built on top of this/// object and the operations it supports.+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]#[derive(Clone, Debug, PartialEq)]pub enum Distribution {Uniform(UniformDistribution),@@ -214,6 +228,10 @@ impl Distribution {////// <https://en.wikipedia.org/wiki/Continuous_uniform_distribution>/// <https://en.wikipedia.org/wiki/Prior_probability#Improper_priors>+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]#[derive(Clone, Debug, PartialEq)]pub struct UniformDistribution {interval: Interval,@@ -236,6 +254,10 @@ pub struct UniformDistribution {/// For more information, see:////// <https://en.wikipedia.org/wiki/Exponential_distribution>+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]#[derive(Clone, Debug, PartialEq)]pub struct ExponentialDistribution {rate: ScalarValue,@@ -249,6 +271,10 @@ pub struct ExponentialDistribution {/// For a more in-depth discussion, see:////// <https://en.wikipedia.org/wiki/Normal_distribution>+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]#[derive(Clone, Debug, PartialEq)]pub struct GaussianDistribution {mean: ScalarValue,@@ -259,6 +285,10 @@ pub struct GaussianDistribution {/// the success probability is unknown. For a more in-depth discussion, see:////// <https://en.wikipedia.org/wiki/Bernoulli_distribution>+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]#[derive(Clone, Debug, PartialEq)]pub struct BernoulliDistribution {p: ScalarValue,@@ -268,6 +298,10 @@ pub struct BernoulliDistribution {/// approximated via some summary statistics. For a more in-depth discussion, see:////// <https://en.wikipedia.org/wiki/Summary_statistics>+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]#[derive(Clone, Debug, PartialEq)]pub struct GenericDistribution {mean: ScalarValue,@@ -594,6 +628,10 @@ impl GenericDistribution {/// This function takes a logical operator and two Bernoulli distributions,/// and it returns a new Bernoulli distribution that represents the result of/// the operation. Currently, only `AND` and `OR` operations are supported.+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]pub fn combine_bernoullis(op: &Operator,left: &BernoulliDistribution,@@ -649,6 +687,10 @@ pub fn combine_bernoullis(/// see:////// <https://en.wikipedia.org/wiki/Sum_of_normally_distributed_random_variables>+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]pub fn combine_gaussians(op: &Operator,left: &GaussianDistribution,@@ -673,6 +715,10 @@ pub fn combine_gaussians(/// Expects `op` to be a comparison operator, with `left` and `right` having/// numeric distributions. The resulting distribution has the `Float64` data/// type.+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]pub fn create_bernoulli_from_comparison(op: &Operator,left: &Distribution,@@ -751,6 +797,10 @@ pub fn create_bernoulli_from_comparison(/// given binary operation on two unknown quantities represented by their/// [`Distribution`] objects. The function computes the mean, median and/// variance if possible.+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]pub fn new_generic_from_binary_op(op: &Operator,left: &Distribution,@@ -766,6 +816,10 @@ pub fn new_generic_from_binary_op(/// Computes the mean value for the result of the given binary operation on/// two unknown quantities represented by their [`Distribution`] objects.+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]pub fn compute_mean(op: &Operator,left: &Distribution,@@ -798,6 +852,10 @@ pub fn compute_mean(/// the median is calculable only for addition and subtraction operations on:/// - [`Uniform`] and [`Uniform`] distributions, and/// - [`Gaussian`] and [`Gaussian`] distributions.+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]pub fn compute_median(op: &Operator,left: &Distribution,@@ -835,6 +893,10 @@ pub fn compute_median(/// Computes the variance value for the result of the given binary operation on/// two unknown quantities represented by their [`Distribution`] objects.+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]pub fn compute_variance(op: &Operator,left: &Distribution,diff --git a/datafusion/ffi/src/expr/distribution.rs b/datafusion/ffi/src/expr/distribution.rsindex ca760f16ad..91b6c4ce75 100644--- a/datafusion/ffi/src/expr/distribution.rs+++ b/datafusion/ffi/src/expr/distribution.rs@@ -15,6 +15,13 @@// specific language governing permissions and limitations// under the License.+//! FFI types for the deprecated Statistics V2 [`Distribution`] framework.+//!+//! These FFI types mirror the deprecated probabilistic distribution types.+//! See <https://github.com/apache/datafusion/pull/22071> for details.++#![allow(deprecated)]+use datafusion_common::DataFusionError;use datafusion_expr::statistics::{BernoulliDistribution, Distribution, ExponentialDistribution, GaussianDistribution,diff --git a/datafusion/ffi/src/physical_expr/mod.rs b/datafusion/ffi/src/physical_expr/mod.rsindex 8756ed2bdc..1125b647f4 100644--- a/datafusion/ffi/src/physical_expr/mod.rs+++ b/datafusion/ffi/src/physical_expr/mod.rs@@ -31,6 +31,7 @@ use datafusion_common::{Result, ffi_datafusion_err};use datafusion_expr::ColumnarValue;use datafusion_expr::interval_arithmetic::Interval;use datafusion_expr::sort_properties::ExprProperties;+#[expect(deprecated)]use datafusion_expr::statistics::Distribution;use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr_common::physical_expr::fmt_sql;@@ -295,6 +296,7 @@ unsafe extern "C" fn propagate_constraints_fn_wrapper(FFI_Result::Ok(result.into())}+#[expect(deprecated)]unsafe extern "C" fn evaluate_statistics_fn_wrapper(expr: &FFI_PhysicalExpr,children: SVec<FFI_Distribution>,@@ -313,6 +315,7 @@ unsafe extern "C" fn evaluate_statistics_fn_wrapper()}+#[expect(deprecated)]unsafe extern "C" fn propagate_statistics_fn_wrapper(expr: &FFI_PhysicalExpr,parent: FFI_Distribution,@@ -630,6 +633,7 @@ impl PhysicalExpr for ForeignPhysicalExpr {}}+ #[expect(deprecated)]fn evaluate_statistics(&self, children: &[&Distribution]) -> Result<Distribution> {unsafe {let children = children@@ -643,6 +647,7 @@ impl PhysicalExpr for ForeignPhysicalExpr {}}+ #[expect(deprecated)]fn propagate_statistics(&self,parent: &Distribution,@@ -739,6 +744,7 @@ mod tests {use datafusion_common::tree_node::DynTreeNode;use datafusion_common::{DataFusionError, ScalarValue};use datafusion_expr::interval_arithmetic::Interval;+ #[expect(deprecated)]use datafusion_expr::statistics::Distribution;use datafusion_physical_expr::expressions::{Column, NegativeExpr, NotExpr};use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, fmt_sql};@@ -879,6 +885,7 @@ mod tests {}#[test]+ #[expect(deprecated)]fn ffi_physical_expr_statistics() -> Result<(), DataFusionError> {let (negative_expr, foreign_neg) = create_test_negative_expr();let interval =diff --git a/datafusion/physical-expr-common/src/physical_expr.rs b/datafusion/physical-expr-common/src/physical_expr.rsindex 6595635024..212dca6cd5 100644--- a/datafusion/physical-expr-common/src/physical_expr.rs+++ b/datafusion/physical-expr-common/src/physical_expr.rs@@ -37,6 +37,7 @@ use datafusion_expr_common::columnar_value::ColumnarValue;use datafusion_expr_common::interval_arithmetic::Interval;use datafusion_expr_common::placement::ExpressionPlacement;use datafusion_expr_common::sort_properties::ExprProperties;+#[expect(deprecated)]use datafusion_expr_common::statistics::Distribution;use itertools::izip;@@ -250,6 +251,11 @@ pub trait PhysicalExpr: Any + Send + Sync + Display + Debug + DynEq + DynHash {/// statistics accordingly. The default implementation simply creates an/// unknown output distribution by combining input ranges. This logic loses/// distribution information, but is a safe default.+ #[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+ )]+ #[expect(deprecated)]fn evaluate_statistics(&self, children: &[&Distribution]) -> Result<Distribution> {let children_ranges = children.iter()@@ -298,6 +304,11 @@ pub trait PhysicalExpr: Any + Send + Sync + Display + Debug + DynEq + DynHash {/// default implementation simply creates an unknown distribution if it can/// narrow the range by propagating ranges. This logic loses distribution/// information, but is a safe default.+ #[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+ )]+ #[expect(deprecated)]fn propagate_statistics(&self,parent: &Distribution,diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rsindex 396c0b87c4..b92668fe9b 100644--- a/datafusion/physical-expr/src/expressions/binary.rs+++ b/datafusion/physical-expr/src/expressions/binary.rs@@ -36,7 +36,9 @@ use datafusion_common::{Result, ScalarValue, internal_err, not_impl_err};use datafusion_expr::binary::BinaryTypeCoercer;use datafusion_expr::interval_arithmetic::{Interval, apply_operator};use datafusion_expr::sort_properties::ExprProperties;+#[expect(deprecated)]use datafusion_expr::statistics::Distribution::{Bernoulli, Gaussian};+#[expect(deprecated)]use datafusion_expr::statistics::{Distribution, combine_bernoullis, combine_gaussians,create_bernoulli_from_comparison, new_generic_from_binary_op,@@ -501,6 +503,7 @@ impl PhysicalExpr for BinaryExpr {}}+ #[expect(deprecated)]fn evaluate_statistics(&self, children: &[&Distribution]) -> Result<Distribution> {let (left, right) = (children[0], children[1]);@@ -4673,6 +4676,7 @@ mod tests {/// Test for Uniform-Uniform, Unknown-Uniform, Uniform-Unknown and Unknown-Unknown evaluation.#[test]+ #[expect(deprecated)]fn test_evaluate_statistics_combination_of_range_holders() -> Result<()> {let schema = &Schema::new(vec![Field::new("a", DataType::Float64, false)]);let a = Arc::new(Column::new("a", 0)) as _;@@ -4740,6 +4744,7 @@ mod tests {}#[test]+ #[expect(deprecated)]fn test_evaluate_statistics_bernoulli() -> Result<()> {let schema = &Schema::new(vec![Field::new("a", DataType::Int64, false),@@ -4775,6 +4780,7 @@ mod tests {}#[test]+ #[expect(deprecated)]fn test_propagate_statistics_combination_of_range_holders_arithmetic() -> Result<()> {let schema = &Schema::new(vec![Field::new("a", DataType::Float64, false)]);let a = Arc::new(Column::new("a", 0)) as _;@@ -4844,6 +4850,7 @@ mod tests {}#[test]+ #[expect(deprecated)]fn test_propagate_statistics_combination_of_range_holders_comparison() -> Result<()> {let schema = &Schema::new(vec![Field::new("a", DataType::Float64, false)]);let a = Arc::new(Column::new("a", 0)) as _;diff --git a/datafusion/physical-expr/src/expressions/negative.rs b/datafusion/physical-expr/src/expressions/negative.rsindex da576f2872..e2bda4c8aa 100644--- a/datafusion/physical-expr/src/expressions/negative.rs+++ b/datafusion/physical-expr/src/expressions/negative.rs@@ -31,6 +31,7 @@ use arrow::{use datafusion_common::{Result, internal_err, plan_err};use datafusion_expr::interval_arithmetic::Interval;use datafusion_expr::sort_properties::ExprProperties;+#[expect(deprecated)]use datafusion_expr::statistics::Distribution::{self, Bernoulli, Exponential, Gaussian, Generic, Uniform,};@@ -134,6 +135,7 @@ impl PhysicalExpr for NegativeExpr {.map(|result| vec![result]))}+ #[expect(deprecated)]fn evaluate_statistics(&self, children: &[&Distribution]) -> Result<Distribution> {match children[0] {Uniform(u) => Distribution::new_uniform(u.range().arithmetic_negate()?),@@ -258,6 +260,7 @@ mod tests {}#[test]+ #[expect(deprecated)]fn test_evaluate_statistics() -> Result<()> {let negative_expr = NegativeExpr::new(Arc::new(Column::new("a", 0)));@@ -337,6 +340,7 @@ mod tests {}#[test]+ #[expect(deprecated)]fn test_propagate_statistics_range_holders() -> Result<()> {let negative_expr = NegativeExpr::new(Arc::new(Column::new("a", 0)));let original_child_interval = Interval::make(Some(-2), Some(3))?;diff --git a/datafusion/physical-expr/src/expressions/not.rs b/datafusion/physical-expr/src/expressions/not.rsindex 917d3a9535..b63effdbb9 100644--- a/datafusion/physical-expr/src/expressions/not.rs+++ b/datafusion/physical-expr/src/expressions/not.rs@@ -28,6 +28,7 @@ use arrow::record_batch::RecordBatch;use datafusion_common::{Result, ScalarValue, cast::as_boolean_array, internal_err};use datafusion_expr::ColumnarValue;use datafusion_expr::interval_arithmetic::Interval;+#[expect(deprecated)]use datafusion_expr::statistics::Distribution::{self, Bernoulli};/// Not expression@@ -126,6 +127,7 @@ impl PhysicalExpr for NotExpr {.map(|result| vec![result]))}+ #[expect(deprecated)]fn evaluate_statistics(&self, children: &[&Distribution]) -> Result<Distribution> {match children[0] {Bernoulli(b) => {@@ -141,6 +143,7 @@ impl PhysicalExpr for NotExpr {}}+ #[expect(deprecated)]fn propagate_statistics(&self,parent: &Distribution,@@ -253,6 +256,7 @@ mod tests {}#[test]+ #[expect(deprecated)]fn test_evaluate_statistics() -> Result<()> {let _schema = &Schema::new(vec![Field::new("a", DataType::Boolean, false)]);let a = Arc::new(Column::new("a", 0)) as _;diff --git a/datafusion/physical-expr/src/statistics/mod.rs b/datafusion/physical-expr/src/statistics/mod.rsindex 02897e0594..115e1b66eb 100644--- a/datafusion/physical-expr/src/statistics/mod.rs+++ b/datafusion/physical-expr/src/statistics/mod.rs@@ -15,6 +15,9 @@// specific language governing permissions and limitations// under the License.-//! Statistics and constraint propagation library+//! Statistics and constraint propagation library.+//!+//! All items exported from this module are **deprecated**;+//! see <https://github.com/apache/datafusion/pull/22071> for details.pub mod stats_solver;diff --git a/datafusion/physical-expr/src/statistics/stats_solver.rs b/datafusion/physical-expr/src/statistics/stats_solver.rsindex 407fa6fd1f..862ff4a032 100644--- a/datafusion/physical-expr/src/statistics/stats_solver.rs+++ b/datafusion/physical-expr/src/statistics/stats_solver.rs@@ -15,6 +15,13 @@// specific language governing permissions and limitations// under the License.+//! DAG-based statistics propagation for the Statistics V2 framework.+//!+//! All public items in this module are **deprecated** as of `54.0.0`.+//! See <https://github.com/apache/datafusion/pull/22071> for details.++#![allow(deprecated)]+use std::sync::Arc;use crate::expressions::Literal;@@ -35,6 +42,10 @@ use petgraph::visit::DfsPostOrder;/// This object implements a directed acyclic expression graph (DAEG) that/// is used to compute statistics/distributions for expressions hierarchically.+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]#[derive(Clone, Debug)]pub struct ExprStatisticsGraph {graph: StableGraph<ExprStatisticsGraphNode, usize>,@@ -43,6 +54,10 @@ pub struct ExprStatisticsGraph {/// This is a node in the DAEG; it encapsulates a reference to the actual/// [`PhysicalExpr`] as well as its statistics/distribution.+#[deprecated(+ since = "54.0.0",+ note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"+)]#[derive(Clone, Debug)]pub struct ExprStatisticsGraphNode {expr: Arc<dyn PhysicalExpr>,