Apache DataFusion SQL Query Engine
test: add NULL values to Fuzz window (#26386)
Nagato Yuzuru91087d474650
diff --git a/datafusion/core/tests/fuzz_cases/window_fuzz.rs b/datafusion/core/tests/fuzz_cases/window_fuzz.rsindex f69b5e9a41..56576241a5 100644--- a/datafusion/core/tests/fuzz_cases/window_fuzz.rs+++ b/datafusion/core/tests/fuzz_cases/window_fuzz.rs@@ -18,7 +18,7 @@use std::sync::Arc;use arrow::array::{ArrayRef, Int32Array, StringArray};-use arrow::compute::{SortOptions, concat_batches};+use arrow::compute::{SortColumn, SortOptions, concat_batches, lexsort_to_indices, take};use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use arrow::util::pretty::pretty_format_batches;@@ -133,10 +133,22 @@ async fn window_bounded_window_random_comparison() -> Result<()> {let mut handles = Vec::new();for i in 0..n {let idx = i % test_cases.len();+ // Change the NULL rate after each pass through `test_cases`, so that+ // each test case runs with each NULL rate.+ let null_pct = NULL_PCTS[(i / test_cases.len()) % NULL_PCTS.len()];+ let sort_options = random_sort_options(i as u64);let (pb_cols, ob_cols, search_mode) = test_cases[idx].clone();let job = SpawnedTask::spawn(run_window_test(- make_staggered_batches::<true>(1000, n_distinct, i as u64),+ make_staggered_batches::<true>(+ 1000,+ n_distinct,+ null_pct,+ sort_options,+ i as u64,+ ),i as u64,+ null_pct,+ sort_options,pb_cols,ob_cols,search_mode,@@ -156,7 +168,15 @@ async fn window_bounded_window_random_comparison() -> Result<()> {async fn bounded_window_causal_non_causal() -> Result<()> {let session_config = SessionConfig::new();let ctx = SessionContext::new_with_config(session_config);- let mut batches = make_staggered_batches::<true>(1000, 10, 23_u64);+ // This test examines only the output batch sizes and has no ORDER BY.+ // NULL values do not change the result of this test.+ let mut batches = make_staggered_batches::<true>(+ 1000,+ 10,+ 0.0,+ [SortOptions::default(); 3],+ 23_u64,+ );// Remove empty batches:batches.retain(|batch| batch.num_rows() > 0);let schema = batches[0].schema();@@ -585,6 +605,8 @@ fn convert_bound_to_current_row_if_applicable(async fn run_window_test(input1: Vec<RecordBatch>,random_seed: u64,+ null_pct: f64,+ sort_options: [SortOptions; 3],partition_by_columns: Vec<&str>,orderby_columns: Vec<&str>,search_mode: InputOrderMode,@@ -596,12 +618,18 @@ async fn run_window_test(let ctx = SessionContext::new_with_config(session_config);let (window_fn, args, fn_name) = get_random_function(&schema, &mut rng, is_linear);let window_frame = get_random_window_frame(&mut rng, is_linear);- let mut orderby_exprs = vec![];- for column in &orderby_columns {- orderby_exprs.push(PhysicalSortExpr {+ // Use the sort options of the input for each column. Then the input+ // satisfies the ordering of the `Sorted` and `PartiallySorted` modes.+ let sort_expr = |column: &str| -> Result<PhysicalSortExpr> {+ let idx = ["a", "b", "c"].iter().position(|c| *c == column).unwrap();+ Ok(PhysicalSortExpr {expr: col(column, &schema)?,- options: SortOptions::default(),+ options: sort_options[idx],})+ };+ let mut orderby_exprs = vec![];+ for column in &orderby_columns {+ orderby_exprs.push(sort_expr(column)?)}if orderby_exprs.len() > 1 && !window_frame.can_accept_multi_orderby() {orderby_exprs.truncate(1);@@ -611,11 +639,8 @@ async fn run_window_test(partitionby_exprs.push(col(column, &schema)?);}let mut sort_keys = vec![];- for partition_by_expr in &partitionby_exprs {- sort_keys.push(PhysicalSortExpr {- expr: partition_by_expr.clone(),- options: SortOptions::default(),- })+ for column in &partition_by_columns {+ sort_keys.push(sort_expr(column)?)}for order_by_expr in &orderby_exprs {if !sort_keys.contains(order_by_expr) {@@ -624,21 +649,8 @@ async fn run_window_test(}let concat_input_record = concat_batches(&schema, &input1)?;- let source_sort_keys: LexOrdering = [- PhysicalSortExpr {- expr: col("a", &schema)?,- options: Default::default(),- },- PhysicalSortExpr {- expr: col("b", &schema)?,- options: Default::default(),- },- PhysicalSortExpr {- expr: col("c", &schema)?,- options: Default::default(),- },- ]- .into();+ let source_sort_keys: LexOrdering =+ [sort_expr("a")?, sort_expr("b")?, sort_expr("c")?].into();let mut exec1 = DataSourceExec::from_data_source(MemorySourceConfig::try_new(&[vec![concat_input_record]], schema.clone(), None)?.try_with_sort_information(vec![source_sort_keys.clone()])?,@@ -699,7 +711,7 @@ async fn run_window_test(// BoundedWindowAggExec should produce more chunk than the usual WindowAggExec.// Otherwise it means that we cannot generate result in running mode.let err_msg = format!(- "Inconsistent result for window_frame: {window_frame:?}, window_fn: {window_fn:?}, args:{args:?}, random_seed: {random_seed:?}, search_mode: {search_mode:?}, partition_by_columns:{partition_by_columns:?}, orderby_columns: {orderby_columns:?}"+ "Inconsistent result for window_frame: {window_frame:?}, window_fn: {window_fn:?}, args:{args:?}, random_seed: {random_seed:?}, null_pct: {null_pct:?}, sort_options: {sort_options:?}, search_mode: {search_mode:?}, partition_by_columns:{partition_by_columns:?}, orderby_columns: {orderby_columns:?}");// Below check makes sure that, streaming execution generates more chunks than the bulk execution.// Since algorithms and operators works on sliding windows in the streaming execution.@@ -733,7 +745,7 @@ async fn run_window_test({if !usual_line.eq(running_line) {println!(- "Inconsistent result for window_frame at line:{i:?}: {window_frame:?}, window_fn: {window_fn:?}, args:{args:?}, pb_cols:{partition_by_columns:?}, ob_cols:{orderby_columns:?}, search_mode:{search_mode:?}"+ "Inconsistent result for window_frame at line:{i:?}: {window_frame:?}, window_fn: {window_fn:?}, args:{args:?}, random_seed: {random_seed:?}, null_pct: {null_pct:?}, sort_options: {sort_options:?}, pb_cols:{partition_by_columns:?}, ob_cols:{orderby_columns:?}, search_mode:{search_mode:?}");println!("--------usual_formatted_sorted----------------running_formatted_sorted--------"@@ -749,6 +761,15 @@ async fn run_window_test(Ok(())}+/// NULL rates for the random window tests+const NULL_PCTS: [f64; 4] = [0.0, 0.01, 0.1, 0.5];++/// Return a random sort direction and NULL position for 'a', 'b' and 'c'+fn random_sort_options(seed: u64) -> [SortOptions; 3] {+ let mut rng = StdRng::seed_from_u64(seed);+ std::array::from_fn(|_| SortOptions::new(rng.random_bool(0.5), rng.random_bool(0.5)))+}+fn generate_random_string(rng: &mut StdRng, length: usize) -> String {rng.sample_iter(&Alphanumeric).take(length)@@ -759,40 +780,56 @@ fn generate_random_string(rng: &mut StdRng, length: usize) -> String {/// Return randomly sized record batches with:/// three sorted int32 columns 'a', 'b', 'c' ranged from 0..DISTINCT as columns/// one random int32 column x+///+/// `sort_options` sets the sort order of 'a', 'b' and 'c', in this sequence.+/// Each value in 'a', 'b', 'c' and 'x' is NULL with the probability `null_pct`.pub(crate) fn make_staggered_batches<const STREAM: bool>(len: usize,n_distinct: usize,+ null_pct: f64,+ sort_options: [SortOptions; 3],random_seed: u64,) -> Vec<RecordBatch> {// use a random number generator to pick a random sized outputlet mut rng = StdRng::seed_from_u64(random_seed);- let mut input123: Vec<(i32, i32, i32)> = vec![(0, 0, 0); len];- let mut input4: Vec<i32> = vec![0; len];+ let mut input4: Vec<Option<i32>> = vec![None; len];let mut input5: Vec<String> = vec!["".to_string(); len];- for v in &mut input123 {- *v = (- rng.random_range(0..n_distinct) as i32,- rng.random_range(0..n_distinct) as i32,- rng.random_range(0..n_distinct) as i32,- )+ let random_key = |rng: &mut StdRng| {+ let value = rng.random_range(0..n_distinct) as i32;+ (!rng.random_bool(null_pct)).then_some(value)+ };+ let keys: [ArrayRef; 3] = std::array::from_fn(|_| {+ Arc::new(Int32Array::from_iter(+ (0..len).map(|_| random_key(&mut rng)),+ )) as _+ });+ for v in &mut input4 {+ let value = rng.random();+ *v = (!rng.random_bool(null_pct)).then_some(value);}- input123.sort_unstable();- rng.fill(&mut input4[..]);for v in &mut input5 {*v = generate_random_string(&mut rng, 1);}input5.sort();- let input1 = Int32Array::from_iter_values(input123.iter().map(|k| k.0));- let input2 = Int32Array::from_iter_values(input123.iter().map(|k| k.1));- let input3 = Int32Array::from_iter_values(input123.iter().map(|k| k.2));- let input4 = Int32Array::from_iter_values(input4);+ let sort_columns: Vec<SortColumn> = keys+ .iter()+ .zip(sort_options)+ .map(|(values, options)| SortColumn {+ values: Arc::clone(values),+ options: Some(options),+ })+ .collect();+ let indices = lexsort_to_indices(&sort_columns, None).unwrap();+ let [input1, input2, input3] =+ keys.map(|values| take(values.as_ref(), &indices, None).unwrap());+ let input4 = Int32Array::from_iter(input4);let input5 = StringArray::from_iter_values(input5);// split into several record batcheslet mut remainder = RecordBatch::try_from_iter(vec![- ("a", Arc::new(input1) as ArrayRef),- ("b", Arc::new(input2) as ArrayRef),- ("c", Arc::new(input3) as ArrayRef),+ ("a", input1),+ ("b", input2),+ ("c", input3),("x", Arc::new(input4) as ArrayRef),("string_field", Arc::new(input5) as ArrayRef),])