Apache DataFusion SQL Query Engine
test: generate NULLs in equivalence fuzz tests (#26086)
Nagato Yuzurue8940acd4ab0
diff --git a/datafusion/core/tests/fuzz_cases/equivalence/ordering.rs b/datafusion/core/tests/fuzz_cases/equivalence/ordering.rsindex 07c416267a..551d027999 100644--- a/datafusion/core/tests/fuzz_cases/equivalence/ordering.rs+++ b/datafusion/core/tests/fuzz_cases/equivalence/ordering.rs@@ -3,22 +3,27 @@// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreementsuse crate::fuzz_cases::equivalence::utils::{- TestScalarUDF, contains_overflowable_arithmetic, create_random_schema,- create_test_params, create_test_schema_2, generate_table_for_eq_properties,- generate_table_for_orderings, is_table_same_after_sort,+ NULL_PCTS, TestScalarUDF, assert_random_ordering_satisfy_is_sound,+ contains_conservative_ordering_op, create_random_schema, create_test_params,+ create_test_schema_2, generate_table_for_eq_properties, generate_table_for_orderings,+ is_table_same_after_sort,};-use arrow::compute::SortOptions;-use datafusion_common::Result;+use arrow::array::{ArrayRef, Float64Array, RecordBatch, new_null_array};+use arrow::compute::{SortOptions, sort};+use arrow::datatypes::{DataType, Field, Schema};use datafusion_common::config::ConfigOptions;+use datafusion_common::{Result, ScalarValue};use datafusion_expr::{Operator, ScalarUDF};-use datafusion_physical_expr::ScalarFunctionExpr;use datafusion_physical_expr::equivalence::{convert_to_orderings, convert_to_sort_exprs,};-use datafusion_physical_expr::expressions::{BinaryExpr, col};+use datafusion_physical_expr::expressions::{BinaryExpr, col, lit};+use datafusion_physical_expr::{EquivalenceProperties, ScalarFunctionExpr};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use itertools::Itertools;+use rand::rngs::StdRng;+use rand::{Rng, SeedableRng};use std::sync::Arc;#[test]@@ -31,12 +36,16 @@ fn test_ordering_satisfy_with_equivalence_random() -> Result<()> {nulls_first: false,};- for seed in 0..N_RANDOM_SCHEMA {+ for (seed, &null_pct) in (0..N_RANDOM_SCHEMA).cartesian_product(NULL_PCTS) {// Create a random schema with random properties- let (test_schema, eq_properties) = create_random_schema(seed as u64)?;+ let (test_schema, eq_properties) = create_random_schema(seed as u64, null_pct)?;// Generate a data that satisfies properties given- let table_data_with_properties =- generate_table_for_eq_properties(&eq_properties, N_ELEMENTS, N_DISTINCT)?;+ let table_data_with_properties = generate_table_for_eq_properties(+ &eq_properties,+ N_ELEMENTS,+ N_DISTINCT,+ null_pct,+ )?;let col_exprs = [col("a", &test_schema)?,col("b", &test_schema)?,@@ -46,8 +55,16 @@ fn test_ordering_satisfy_with_equivalence_random() -> Result<()> {col("f", &test_schema)?,];+ let mut options_rng = StdRng::seed_from_u64(seed as u64);for n_req in 1..=col_exprs.len() {for exprs in col_exprs.iter().combinations(n_req) {+ assert_random_ordering_satisfy_is_sound(+ &eq_properties,+ &exprs,+ &table_data_with_properties,+ &mut options_rng,+ &format!("seed: {seed}, null_pct: {null_pct}"),+ )?;let sort_exprs = exprs.into_iter().map(|expr| PhysicalSortExpr::new(Arc::clone(expr), SORT_OPTIONS));@@ -59,7 +76,7 @@ fn test_ordering_satisfy_with_equivalence_random() -> Result<()> {&table_data_with_properties,)?;let err_msg = format!(- "Error in test case requirement:{ordering:?}, expected: {expected:?}, eq_properties {eq_properties}"+ "Error in test case seed: {seed}, null_pct: {null_pct}, requirement:{ordering:?}, expected: {expected:?}, eq_properties {eq_properties}");// Check whether ordering_satisfy API result and// experimental result matches.@@ -75,6 +92,19 @@ fn test_ordering_satisfy_with_equivalence_random() -> Result<()> {Ok(())}+const N_EXHAUSTIVE_EXPRS: usize = 8;+const SAMPLED_COMBINATION_PROBABILITY: f64 = 0.03;++/// Selects all combinations of the first `N_EXHAUSTIVE_EXPRS` expressions and+/// all combinations of at most two expressions, and samples the others.+fn is_combination_selected(indices: &[usize], rng: &mut StdRng) -> bool {+ let only_exhaustive_exprs =+ indices.last().is_none_or(|&last| last < N_EXHAUSTIVE_EXPRS);+ only_exhaustive_exprs+ || indices.len() <= 2+ || rng.random_bool(SAMPLED_COMBINATION_PROBABILITY)+}+#[test]fn test_ordering_satisfy_with_equivalence_complex_random() -> Result<()> {const N_RANDOM_SCHEMA: usize = 100;@@ -85,12 +115,16 @@ fn test_ordering_satisfy_with_equivalence_complex_random() -> Result<()> {nulls_first: false,};- for seed in 0..N_RANDOM_SCHEMA {+ for (seed, &null_pct) in (0..N_RANDOM_SCHEMA).cartesian_product(NULL_PCTS) {// Create a random schema with random properties- let (test_schema, eq_properties) = create_random_schema(seed as u64)?;+ let (test_schema, eq_properties) = create_random_schema(seed as u64, null_pct)?;// Generate a data that satisfies properties given- let table_data_with_properties =- generate_table_for_eq_properties(&eq_properties, N_ELEMENTS, N_DISTINCT)?;+ let table_data_with_properties = generate_table_for_eq_properties(+ &eq_properties,+ N_ELEMENTS,+ N_DISTINCT,+ null_pct,+ )?;let test_fun = Arc::new(ScalarUDF::new_from_impl(TestScalarUDF::new()));let col_a = col("a", &test_schema)?;@@ -105,6 +139,31 @@ fn test_ordering_satisfy_with_equivalence_complex_random() -> Result<()> {Operator::Plus,col("b", &test_schema)?,)) as Arc<dyn PhysicalExpr>;+ let binary = |lhs: Arc<dyn PhysicalExpr>, op, rhs: Arc<dyn PhysicalExpr>| {+ Arc::new(BinaryExpr::new(lhs, op, rhs)) as Arc<dyn PhysicalExpr>+ };+ // Ordered when `a` and `b` lead orderings in opposite directions.+ let a_gt_b = binary(+ col("a", &test_schema)?,+ Operator::Gt,+ col("b", &test_schema)?,+ );+ // `e` is constant, so each operand has the ordering of its column.+ // `a` and `b` lead separate orderings whose NULLs sit in different+ // rows, which is where Kleene `AND`/`OR` can break an ordering.+ let a_gt_e = binary(+ col("a", &test_schema)?,+ Operator::Gt,+ col("e", &test_schema)?,+ );+ let b_gt_e = binary(+ col("b", &test_schema)?,+ Operator::Gt,+ col("e", &test_schema)?,+ );+ let a_gt_e_and_b_gt_e =+ binary(Arc::clone(&a_gt_e), Operator::And, Arc::clone(&b_gt_e));+ let a_gt_e_or_b_gt_e = binary(a_gt_e, Operator::Or, b_gt_e);let exprs = [col("a", &test_schema)?,col("b", &test_schema)?,@@ -114,10 +173,26 @@ fn test_ordering_satisfy_with_equivalence_complex_random() -> Result<()> {col("f", &test_schema)?,floor_a,a_plus_b,+ a_gt_b,+ a_gt_e_and_b_gt_e,+ a_gt_e_or_b_gt_e,];+ let mut options_rng = StdRng::seed_from_u64(seed as u64);+ let mut selection_rng = StdRng::seed_from_u64(!(seed as u64));for n_req in 1..=exprs.len() {- for exprs in exprs.iter().combinations(n_req) {+ for indices in (0..exprs.len()).combinations(n_req) {+ if !is_combination_selected(&indices, &mut selection_rng) {+ continue;+ }+ let exprs = indices.iter().map(|&i| &exprs[i]).collect::<Vec<_>>();+ assert_random_ordering_satisfy_is_sound(+ &eq_properties,+ &exprs,+ &table_data_with_properties,+ &mut options_rng,+ &format!("seed: {seed}, null_pct: {null_pct}"),+ )?;let sort_exprs = exprs.into_iter().map(|expr| PhysicalSortExpr::new(Arc::clone(expr), SORT_OPTIONS));@@ -129,19 +204,20 @@ fn test_ordering_satisfy_with_equivalence_complex_random() -> Result<()> {&table_data_with_properties,)?;let err_msg = format!(- "Error in test case requirement:{ordering:?}, expected: {expected:?}, eq_properties: {eq_properties}",+ "Error in test case seed: {seed}, null_pct: {null_pct}, requirement:{ordering:?}, expected: {expected:?}, eq_properties: {eq_properties}",);- // A rejection turns inconclusive only from the first `+`/`-`- // key onwards, since possible overflow makes an ordering- // underivable even when the sample happens to be sorted. A- // table sorted by the full ordering is sorted by every prefix- // of it, so a rejected arithmetic-free prefix still proves- // the rejection is genuine.+ // A rejection turns inconclusive only from the first key with+ // a conservative ordering rule onwards (see+ // `contains_conservative_ordering_op`), since such a rule can+ // make an ordering underivable even when the sample happens+ // to be sorted. A table sorted by the full ordering is sorted+ // by every prefix of it, so a rejected prefix without such+ // keys still proves the rejection is genuine.let conclusive_prefix = LexOrdering::new(ordering.iter().take_while(|sort_expr| {- !contains_overflowable_arithmetic(&sort_expr.expr)+ !contains_conservative_ordering_op(&sort_expr.expr)}).cloned(),);@@ -159,6 +235,140 @@ fn test_ordering_satisfy_with_equivalence_complex_random() -> Result<()> {Ok(())}+const ALL_SORT_OPTIONS: [SortOptions; 4] = [+ SortOptions {+ descending: false,+ nulls_first: false,+ },+ SortOptions {+ descending: false,+ nulls_first: true,+ },+ SortOptions {+ descending: true,+ nulls_first: false,+ },+ SortOptions {+ descending: true,+ nulls_first: true,+ },+];++/// Returns `0.0..=5.0` and `n_nulls` NULLs, sorted with `options`.+fn sorted_f64_array(options: SortOptions, n_nulls: usize) -> Result<ArrayRef> {+ let values = (0..6)+ .map(|v| Some(v as f64))+ .chain(std::iter::repeat_n(None, n_nulls))+ .collect::<Float64Array>();+ Ok(sort(&values, Some(options))?)+}++/// Returns whether `eq_properties` claims the ordering of `exprs` with+/// `options`, and asserts that `batch` has every claimed ordering.+fn assert_claim_is_sound(+ eq_properties: &EquivalenceProperties,+ exprs: &[&Arc<dyn PhysicalExpr>],+ options: SortOptions,+ batch: &RecordBatch,+) -> Result<bool> {+ let sort_exprs = exprs+ .iter()+ .map(|expr| PhysicalSortExpr::new(Arc::clone(expr), options));+ let Some(ordering) = LexOrdering::new(sort_exprs) else {+ unreachable!("Test should always produce non-degenerate orderings");+ };+ let claimed = eq_properties.ordering_satisfy(ordering.clone())?;+ if claimed {+ assert!(+ is_table_same_after_sort(ordering.clone(), batch)?,+ "requirement: {ordering:?}, eq_properties: {eq_properties}"+ );+ }+ Ok(claimed)+}++#[test]+fn test_ordering_satisfy_all_null_ordered_column() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Float64, true),+ Field::new("b", DataType::Float64, true),+ ]));+ let col_a = &col("a", &schema)?;+ let col_b = &col("b", &schema)?;+ for options in ALL_SORT_OPTIONS {+ let mut eq_properties = EquivalenceProperties::new(Arc::clone(&schema));+ eq_properties.add_ordering([+ PhysicalSortExpr::new(Arc::clone(col_a), options),+ PhysicalSortExpr::new(Arc::clone(col_b), options),+ ]);+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ new_null_array(&DataType::Float64, 6),+ sorted_f64_array(options, 0)?,+ ],+ )?;+ for required in ALL_SORT_OPTIONS {+ for exprs in [+ vec![col_a],+ vec![col_b],+ vec![col_a, col_b],+ vec![col_b, col_a],+ ] {+ assert_claim_is_sound(&eq_properties, &exprs, required, &batch)?;+ }+ }+ for exprs in [vec![col_a], vec![col_a, col_b]] {+ assert!(+ assert_claim_is_sound(&eq_properties, &exprs, options, &batch)?,+ "requirement: {exprs:?} {options:?}, eq_properties: {eq_properties}"+ );+ }+ }+ Ok(())+}++#[test]+fn test_ordering_satisfy_null_constant_operand() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, true)]));+ let col_a = col("a", &schema)?;+ // `x` has the ordering of `a`.+ let x =+ Arc::new(BinaryExpr::new(col_a, Operator::Gt, lit(2.0))) as Arc<dyn PhysicalExpr>;+ let null = lit(ScalarValue::Boolean(None));+ let x_and_null = Arc::new(BinaryExpr::new(+ Arc::clone(&x),+ Operator::And,+ Arc::clone(&null),+ )) as Arc<dyn PhysicalExpr>;+ let x_or_null = Arc::new(BinaryExpr::new(Arc::clone(&x), Operator::Or, null))+ as Arc<dyn PhysicalExpr>;+ for options in ALL_SORT_OPTIONS {+ let mut eq_properties = EquivalenceProperties::new(Arc::clone(&schema));+ eq_properties.add_ordering([PhysicalSortExpr::new(col("a", &schema)?, options)]);+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![sorted_f64_array(options, 2)?],+ )?;+ assert!(+ assert_claim_is_sound(&eq_properties, &[&x], options, &batch)?,+ "requirement: {x} {options:?}, eq_properties: {eq_properties}"+ );+ // A NULL operand moves rows to the NULL position, so `AND` and `OR`+ // keep the ordering only for some NULL positions.+ let mut n_claimed = 0;+ for expr in [&x_and_null, &x_or_null] {+ for required in ALL_SORT_OPTIONS {+ if assert_claim_is_sound(&eq_properties, &[expr], required, &batch)? {+ n_claimed += 1;+ }+ }+ }+ assert!(n_claimed > 0, "no claim with {options:?}");+ }+ Ok(())+}+#[test]fn test_ordering_satisfy_with_equivalence() -> Result<()> {// Schema satisfies following orderings:@@ -184,7 +394,7 @@ fn test_ordering_satisfy_with_equivalence() -> Result<()> {nulls_first: true,};let table_data_with_properties =- generate_table_for_eq_properties(&eq_properties, 625, 5)?;+ generate_table_for_eq_properties(&eq_properties, 625, 5, 0.0)?;// First element in the tuple stores vector of requirement, second element is the expected return value for ordering_satisfy functionlet requirements = vec![@@ -359,7 +569,7 @@ fn test_ordering_satisfy_on_data() -> Result<()> {];let orderings = convert_to_orderings(&orderings);- let batch = generate_table_for_orderings(orderings, schema, 1000, 10)?;+ let batch = generate_table_for_orderings(orderings, schema, 1000, 10, 0.0)?;// [a ASC, c ASC, d ASC] cannot be deducedlet ordering = vec![diff --git a/datafusion/core/tests/fuzz_cases/equivalence/projection.rs b/datafusion/core/tests/fuzz_cases/equivalence/projection.rsindex d3a7cc83e9..15b6d26eef 100644--- a/datafusion/core/tests/fuzz_cases/equivalence/projection.rs+++ b/datafusion/core/tests/fuzz_cases/equivalence/projection.rs@@ -3,8 +3,9 @@// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreementsuse crate::fuzz_cases::equivalence::utils::{- TestScalarUDF, apply_projection, contains_overflowable_arithmetic,- create_random_schema, generate_table_for_eq_properties, is_table_same_after_sort,+ NULL_PCTS, TestScalarUDF, apply_projection, assert_random_ordering_satisfy_is_sound,+ contains_conservative_ordering_op, create_random_schema,+ generate_table_for_eq_properties, is_table_same_after_sort,};use arrow::compute::SortOptions;use datafusion_common::Result;@@ -16,6 +17,8 @@ use datafusion_physical_expr::{PhysicalExprRef, ScalarFunctionExpr};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use itertools::Itertools;+use rand::SeedableRng;+use rand::rngs::StdRng;use std::sync::Arc;#[test]@@ -24,12 +27,16 @@ fn project_orderings_random() -> Result<()> {const N_ELEMENTS: usize = 125;const N_DISTINCT: usize = 5;- for seed in 0..N_RANDOM_SCHEMA {+ for (seed, &null_pct) in (0..N_RANDOM_SCHEMA).cartesian_product(NULL_PCTS) {// Create a random schema with random properties- let (test_schema, eq_properties) = create_random_schema(seed as u64)?;+ let (test_schema, eq_properties) = create_random_schema(seed as u64, null_pct)?;// Generate a data that satisfies properties given- let table_data_with_properties =- generate_table_for_eq_properties(&eq_properties, N_ELEMENTS, N_DISTINCT)?;+ let table_data_with_properties = generate_table_for_eq_properties(+ &eq_properties,+ N_ELEMENTS,+ N_DISTINCT,+ null_pct,+ )?;// Floor(a)let test_fun = Arc::new(ScalarUDF::new_from_impl(TestScalarUDF::new()));let col_a = col("a", &test_schema)?;@@ -71,7 +78,7 @@ fn project_orderings_random() -> Result<()> {// Make sure each ordering after projection is valid.for ordering in projected_eq.oeq_class().iter() {let err_msg = format!(- "Error in test case ordering:{ordering:?}, eq_properties {eq_properties}, proj_exprs: {proj_exprs:?}",+ "Error in test case seed: {seed}, null_pct: {null_pct}, ordering:{ordering:?}, eq_properties {eq_properties}, proj_exprs: {proj_exprs:?}",);// Since ordered section satisfies schema, we expect// that result will be same after sort (e.g sort was unnecessary).@@ -98,12 +105,16 @@ fn ordering_satisfy_after_projection_random() -> Result<()> {nulls_first: false,};- for seed in 0..N_RANDOM_SCHEMA {+ for (seed, &null_pct) in (0..N_RANDOM_SCHEMA).cartesian_product(NULL_PCTS) {// Create a random schema with random properties- let (test_schema, eq_properties) = create_random_schema(seed as u64)?;+ let (test_schema, eq_properties) = create_random_schema(seed as u64, null_pct)?;// Generate a data that satisfies properties given- let table_data_with_properties =- generate_table_for_eq_properties(&eq_properties, N_ELEMENTS, N_DISTINCT)?;+ let table_data_with_properties = generate_table_for_eq_properties(+ &eq_properties,+ N_ELEMENTS,+ N_DISTINCT,+ null_pct,+ )?;// Floor(a)let test_fun = Arc::new(ScalarUDF::new_from_impl(TestScalarUDF::new()));let col_a = col("a", &test_schema)?;@@ -130,6 +141,7 @@ fn ordering_satisfy_after_projection_random() -> Result<()> {(a_plus_b, "a+b"),];+ let mut options_rng = StdRng::seed_from_u64(seed as u64);for n_req in 0..=proj_exprs.len() {for proj_exprs in proj_exprs.iter().combinations(n_req) {let proj_exprs = proj_exprs@@ -153,6 +165,13 @@ fn ordering_satisfy_after_projection_random() -> Result<()> {for n_req in 1..=projected_exprs.len() {for exprs in projected_exprs.iter().combinations(n_req) {+ assert_random_ordering_satisfy_is_sound(+ &projected_eq,+ &exprs,+ &projected_batch,+ &mut options_rng,+ &format!("seed: {seed}, null_pct: {null_pct}"),+ )?;let sort_exprs = exprs.into_iter().map(|expr| {PhysicalSortExpr::new(Arc::clone(expr), SORT_OPTIONS)});@@ -164,10 +183,11 @@ fn ordering_satisfy_after_projection_random() -> Result<()> {let expected =is_table_same_after_sort(ordering.clone(), &projected_batch)?;let err_msg = format!(- "Error in test case requirement:{ordering:?}, expected: {expected:?}, eq_properties: {eq_properties}, projected_eq: {projected_eq}, projection_mapping: {projection_mapping:?}"+ "Error in test case seed: {seed}, null_pct: {null_pct}, requirement:{ordering:?}, expected: {expected:?}, eq_properties: {eq_properties}, projected_eq: {projected_eq}, projection_mapping: {projection_mapping:?}");// Same reasoning as in `ordering.rs`: only keys from- // the first `+`/`-` source onwards are inconclusive,+ // the first source with a conservative ordering rule+ // onwards are inconclusive,// so assert on the longest prefix without one.let conclusive_prefix = LexOrdering::new(ordering@@ -177,7 +197,7 @@ fn ordering_satisfy_after_projection_random() -> Result<()> {targets.iter().any(|(target, _)| target.eq(&sort_expr.expr))- && contains_overflowable_arithmetic(source)+ && contains_conservative_ordering_op(source)})}).cloned(),diff --git a/datafusion/core/tests/fuzz_cases/equivalence/properties.rs b/datafusion/core/tests/fuzz_cases/equivalence/properties.rsindex 3fc205f0ed..1d23335f39 100644--- a/datafusion/core/tests/fuzz_cases/equivalence/properties.rs+++ b/datafusion/core/tests/fuzz_cases/equivalence/properties.rs@@ -5,7 +5,7 @@use std::sync::Arc;use crate::fuzz_cases::equivalence::utils::{- TestScalarUDF, create_random_schema, generate_table_for_eq_properties,+ NULL_PCTS, TestScalarUDF, create_random_schema, generate_table_for_eq_properties,is_table_same_after_sort,};@@ -24,12 +24,16 @@ fn test_find_longest_permutation_random() -> Result<()> {const N_ELEMENTS: usize = 125;const N_DISTINCT: usize = 5;- for seed in 0..N_RANDOM_SCHEMA {+ for (seed, &null_pct) in (0..N_RANDOM_SCHEMA).cartesian_product(NULL_PCTS) {// Create a random schema with random properties- let (test_schema, eq_properties) = create_random_schema(seed as u64)?;+ let (test_schema, eq_properties) = create_random_schema(seed as u64, null_pct)?;// Generate a data that satisfies properties given- let table_data_with_properties =- generate_table_for_eq_properties(&eq_properties, N_ELEMENTS, N_DISTINCT)?;+ let table_data_with_properties = generate_table_for_eq_properties(+ &eq_properties,+ N_ELEMENTS,+ N_DISTINCT,+ null_pct,+ )?;let test_fun = Arc::new(ScalarUDF::new_from_impl(TestScalarUDF::new()));let col_a = col("a", &test_schema)?;@@ -75,7 +79,7 @@ fn test_find_longest_permutation_random() -> Result<()> {);let err_msg = format!(- "Error in test case ordering:{ordering:?}, eq_properties: {eq_properties}"+ "Error in test case seed: {seed}, null_pct: {null_pct}, ordering:{ordering:?}, eq_properties: {eq_properties}");assert_eq!(ordering.len(), indices.len(), "{err_msg}");// Since ordered section satisfies schema, we expectdiff --git a/datafusion/core/tests/fuzz_cases/equivalence/utils.rs b/datafusion/core/tests/fuzz_cases/equivalence/utils.rsindex 8f0ff8b354..49da985a64 100644--- a/datafusion/core/tests/fuzz_cases/equivalence/utils.rs+++ b/datafusion/core/tests/fuzz_cases/equivalence/utils.rs@@ -7,13 +7,14 @@ use std::sync::Arc;use arrow::array::{ArrayRef, Float32Array, Float64Array, RecordBatch, UInt32Array};use arrow::compute::{SortColumn, SortOptions, lexsort_to_indices, take_record_batch};-use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef};use datafusion_common::tree_node::TreeNode;use datafusion_common::utils::{compare_rows, get_row_at_idx};use datafusion_common::{Result, exec_err, internal_datafusion_err, plan_err};use datafusion_expr::sort_properties::{ExprProperties, SortProperties};use datafusion_expr::{- ColumnarValue, Operator, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,+ ColumnarValue, Operator, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,+ Signature, Volatility,};use datafusion_physical_expr::equivalence::{EquivalenceClass, ProjectionMapping, convert_to_orderings,@@ -70,8 +71,21 @@ pub fn create_test_schema_2() -> Result<SchemaRef> {/// where/// Column [a=f] (e.g they are aliases)./// Column e is constant.-pub fn create_random_schema(seed: u64) -> Result<(SchemaRef, EquivalenceProperties)> {- let test_schema = create_test_schema_2()?;+///+/// Columns are declared nullable only when `null_pct > 0.0`, so the schema+/// tells `ordering_satisfy` whether `nulls_first` can matter for the data.+pub fn create_random_schema(+ seed: u64,+ null_pct: f64,+) -> Result<(SchemaRef, EquivalenceProperties)> {+ let nullable = null_pct > 0.0;+ let test_schema = Arc::new(Schema::new(+ create_test_schema_2()?+ .fields()+ .iter()+ .map(|field| field.as_ref().clone().with_nullable(nullable))+ .collect::<Vec<_>>(),+ ));let col_a = &col("a", &test_schema)?;let col_b = &col("b", &test_schema)?;let col_c = &col("c", &test_schema)?;@@ -90,11 +104,6 @@ pub fn create_random_schema(seed: u64) -> Result<(SchemaRef, EquivalencePropertilet mut rng = StdRng::seed_from_u64(seed);let mut remaining_exprs = col_exprs[0..4].to_vec(); // only a, b, c, d are sorted- let options_asc = SortOptions {- descending: false,- nulls_first: false,- };-while !remaining_exprs.is_empty() {let n_sort_expr = rng.random_range(1..remaining_exprs.len() + 1);remaining_exprs.shuffle(&mut rng);@@ -104,7 +113,7 @@ pub fn create_random_schema(seed: u64) -> Result<(SchemaRef, EquivalenceProperti.drain(0..n_sort_expr).map(|expr| PhysicalSortExpr {expr: Arc::clone(expr),- options: options_asc,+ options: random_sort_options(&mut rng),});eq_properties.add_ordering(ordering);@@ -113,6 +122,14 @@ pub fn create_random_schema(seed: u64) -> Result<(SchemaRef, EquivalencePropertiOk((test_schema, eq_properties))}+/// Picks the direction and NULL placement of one sort key.+pub fn random_sort_options(rng: &mut StdRng) -> SortOptions {+ SortOptions {+ descending: rng.random(),+ nulls_first: rng.random(),+ }+}+// Apply projection to the input_data, return projected equivalence properties and record batchpub fn apply_projection(proj_exprs: impl IntoIterator<Item = (Arc<dyn PhysicalExpr>, String)>,@@ -197,15 +214,29 @@ fn add_equal_conditions_test() -> Result<()> {Ok(())}-/// Returns `true` if `expr` contains a `+` or `-` anywhere in its tree.+/// Returns `true` if `expr` contains, anywhere in its tree, an operator whose+/// ordering rule is deliberately conservative.///-/// The equivalence framework conservatively discards orderings derived from-/// `+`/`-` expressions, because wrapping overflow can break them over the-/// type's full domain even when a finite batch happens to remain sorted.-pub fn contains_overflowable_arithmetic(expr: &Arc<dyn PhysicalExpr>) -> bool {+/// The equivalence framework discards orderings in cases where a finite batch+/// can still happen to be sorted, so a rejection is not conclusive:+/// - `+`/`-`: wrapping overflow can break the ordering over the type's full+/// domain.+/// - comparisons, `AND`, `OR`: the rules keep an ordering only for specific+/// NULL placements, without checking whether the operands can be NULL.+pub fn contains_conservative_ordering_op(expr: &Arc<dyn PhysicalExpr>) -> bool {expr.exists(|e| {Ok(e.downcast_ref::<BinaryExpr>().is_some_and(|binary| {- matches!(binary.op(), Operator::Plus | Operator::Minus)+ matches!(+ binary.op(),+ Operator::Plus+ | Operator::Minus+ | Operator::Gt+ | Operator::GtEq+ | Operator::Lt+ | Operator::LtEq+ | Operator::And+ | Operator::Or+ )}))}).unwrap()@@ -275,6 +306,34 @@ pub fn is_table_same_after_sort(Ok(sorted_indices == original_indices)}+/// Sorts `exprs` with random [`SortOptions`] and asserts that `eq_properties`+/// never claims an ordering that `batch` does not have.+///+/// Only this direction is checked: a single batch cannot tell an underivable+/// ordering from one that is sorted by coincidence, e.g. when NULLs of+/// independently sorted columns end up in the same rows.+pub fn assert_random_ordering_satisfy_is_sound(+ eq_properties: &EquivalenceProperties,+ exprs: &[&Arc<dyn PhysicalExpr>],+ batch: &RecordBatch,+ rng: &mut StdRng,+ context: &str,+) -> Result<()> {+ let sort_exprs = exprs+ .iter()+ .map(|expr| PhysicalSortExpr::new(Arc::clone(expr), random_sort_options(rng)));+ let Some(ordering) = LexOrdering::new(sort_exprs) else {+ unreachable!("Test should always produce non-degenerate orderings");+ };+ if eq_properties.ordering_satisfy(ordering.clone())? {+ assert!(+ is_table_same_after_sort(ordering.clone(), batch)?,+ "{context}, random requirement: {ordering:?}, eq_properties: {eq_properties}"+ );+ }+ Ok(())+}+// If we already generated a random result for one of the// expressions in the equivalence classes. For other expressions in the same// equivalence class use same result. This util gets already calculated result, when available.@@ -356,6 +415,7 @@ pub fn generate_table_for_eq_properties(eq_properties: &EquivalenceProperties,n_elem: usize,n_distinct: usize,+ null_pct: f64,) -> Result<RecordBatch> {let mut rng = StdRng::seed_from_u64(23);@@ -364,10 +424,7 @@ pub fn generate_table_for_eq_properties(// Utility closure to generate random arraylet mut generate_random_array = |num_elems: usize, max_val: usize| -> ArrayRef {- let values: Vec<f64> = (0..num_elems)- .map(|_| rng.random_range(0..max_val) as f64 / 2.0)- .collect();- Arc::new(Float64Array::from_iter_values(values))+ generate_random_f64_array(num_elems, max_val, null_pct, &mut rng)};// Fill constant columns@@ -437,6 +494,7 @@ pub fn generate_table_for_orderings(schema: SchemaRef,n_elem: usize,n_distinct: usize,+ null_pct: f64,) -> Result<RecordBatch> {let mut rng = StdRng::seed_from_u64(23);@@ -450,7 +508,7 @@ pub fn generate_table_for_orderings(.map(|field| {(field.name(),- generate_random_f64_array(n_elem, n_distinct, &mut rng),+ generate_random_f64_array(n_elem, n_distinct, null_pct, &mut rng),)}).collect::<Vec<_>>();@@ -490,16 +548,25 @@ pub fn generate_table_for_orderings(Ok(batch)}+pub const NULL_PCTS: &[f64] = &[0.0, 0.1, 0.5];+// Utility function to generate random f64 arrayfn generate_random_f64_array(n_elems: usize,n_distinct: usize,+ null_pct: f64,rng: &mut StdRng,) -> ArrayRef {- let values: Vec<f64> = (0..n_elems)- .map(|_| rng.random_range(0..n_distinct) as f64 / 2.0)- .collect();- Arc::new(Float64Array::from_iter_values(values))+ let values = (0..n_elems)+ .map(|_| {+ if rng.random::<f64>() < null_pct {+ None+ } else {+ Some(rng.random_range(0..n_distinct) as f64 / 2.0)+ }+ })+ .collect::<Float64Array>();+ Arc::new(values)}// Helper function to get sort columns from a batch@@ -549,6 +616,17 @@ impl ScalarUDFImpl for TestScalarUDF {}}+ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {+ let arg_field = &args.arg_fields[0];+ let return_type = self.return_type(&[arg_field.data_type().clone()])?;+ // floor maps NULL to NULL and never produces NULL otherwise.+ Ok(Arc::new(Field::new(+ self.name(),+ return_type,+ arg_field.is_nullable(),+ )))+ }+fn output_ordering(&self, input: &[ExprProperties]) -> Result<SortProperties> {Ok(input[0].sort_properties)}