Apache DataFusion SQL Query Engine
test: refactor key-range bitmap union fuzz test to FuzzTest trait (#27258)
Mrigank Bhatnagar0142e911fd3f
diff --git a/datafusion/fuzz/tests/fuzz.rs b/datafusion/fuzz/tests/fuzz.rsindex 6eae5f9127..fd4a0ff1a0 100644--- a/datafusion/fuzz/tests/fuzz.rs+++ b/datafusion/fuzz/tests/fuzz.rs@@ -20,15 +20,27 @@ mod suites;#[tokio::test(flavor = "multi_thread")]async fn fuzz_tests() -> Result<()> {let spill_pool = suites::spill_pool::SpillPoolFuzzer;- let suites = [Suite::new(- &spill_pool,- RunConfig {- run_seed: Some(0),- max_cases: Some(1000),- time_limit: None,- cases_per_dataset: 1000,- },- )];+ let key_range_bitmap = suites::key_range_bitmap::KeyRangeBitmapFuzzer;+ let suites = [+ Suite::new(+ &spill_pool,+ RunConfig {+ run_seed: Some(0),+ max_cases: Some(1000),+ time_limit: None,+ cases_per_dataset: 1000,+ },+ ),+ Suite::new(+ &key_range_bitmap,+ RunConfig {+ run_seed: Some(0),+ max_cases: Some(10000),+ time_limit: None,+ cases_per_dataset: 10000,+ },+ ),+ ];run_suites(suites).await}diff --git a/datafusion/fuzz/tests/suites/key_range_bitmap.rs b/datafusion/fuzz/tests/suites/key_range_bitmap.rsnew file mode 100644index 0000000000..c0dbebcd09--- /dev/null+++ b/datafusion/fuzz/tests/suites/key_range_bitmap.rs@@ -0,0 +1,195 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements++//! Randomized partitions against [`KeyRangeBitmap::union`]'s contract: it+//! never excludes a key any part may hold, equals the bitmap built from all+//! parts' keys when no part brings its own bitmap, and only widens that+//! bitmap's buckets when one does.++use std::future::Future;+use std::sync::Arc;++use arrow::array::{ArrayRef, Int64Array};+use arrow::buffer::BooleanBuffer;+use datafusion_common::{Result, ScalarValue};+use datafusion_physical_plan::joins::key_range_bitmap::{KeyRangeBitmap, KeyRangePart};+use rand::rngs::StdRng;+use rand::{Rng, SeedableRng};+use test_utils::fuzz::FuzzTest;++pub(crate) struct KeyRangeBitmapFuzzer;++fn parts_of(b: &KeyRangeBitmap) -> (u64, u64, u32, &BooleanBuffer) {+ (b.offset(), b.span(), b.shift(), b.bits())+}++enum Part {+ Keys(ArrayRef),+ Range(ScalarValue, ScalarValue),+ Bitmap(KeyRangeBitmap),+}++impl FuzzTest for KeyRangeBitmapFuzzer {+ type Dataset = ();++ fn name(&self) -> &str {+ "key_range_bitmap_union"+ }++ fn prepare_dataset(&self, _seed: u64) -> impl Future<Output = Result<()>> {+ std::future::ready(Ok(()))+ }++ fn run_case(&self, _dataset: &(), seed: u64) -> impl Future<Output = Result<()>> {+ let mut rng = StdRng::seed_from_u64(seed);+ // Domain widths from one key per bucket to many keys per bucket, so+ // the union's buckets are sometimes exact and sometimes coarse.+ let width = [64i64, 1 << 12, 1 << 24, 1 << 40][rng.random_range(0..4)];+ let start = rng.random_range(-(1i64 << 50)..(1i64 << 50));+ let key = |rng: &mut StdRng| start + rng.random_range(0..width);+ let keys_array =+ |keys: &[i64]| -> ArrayRef { Arc::new(Int64Array::from(keys.to_vec())) };+ let scalar = |key: i64| ScalarValue::Int64(Some(key));++ // Each part, and every key any part may hold.+ let mut parts = vec![];+ let mut all_keys = vec![];+ let mut has_bitmap = false;+ for _ in 0..rng.random_range(1..5) {+ if rng.random_bool(1.0 / 3.0) {+ // Narrow, so every key it may hold can be listed.+ let lo = key(&mut rng);+ let hi = lo + rng.random_range(0..64);+ parts.push(Part::Range(scalar(lo), scalar(hi)));+ all_keys.extend(lo..=hi);+ continue;+ }+ let keys: Vec<i64> = (0..rng.random_range(1..50))+ .map(|_| key(&mut rng))+ .collect();+ let (lo, hi) = (*keys.iter().min().unwrap(), *keys.iter().max().unwrap());+ let bitmap = rng+ .random_bool(0.5)+ .then(|| {+ KeyRangeBitmap::try_new(+ &keys_array(&keys),+ &scalar(lo),+ &scalar(hi),+ 0,+ )+ })+ .flatten();+ parts.push(match bitmap {+ Some(bitmap) => {+ has_bitmap = true;+ Part::Bitmap(bitmap)+ }+ None => Part::Keys(keys_array(&keys)),+ });+ all_keys.extend(keys);+ }+ let min = ScalarValue::Int64(all_keys.iter().min().copied());+ let max = ScalarValue::Int64(all_keys.iter().max().copied());+ let context = format!(+ "seed {seed}: width {width}, {} parts, {} keys",+ parts.len(),+ all_keys.len()+ );++ let union = KeyRangeBitmap::union(+ &min,+ &max,+ &parts+ .iter()+ .map(|part| match part {+ Part::Keys(keys) => KeyRangePart::Keys(keys.as_ref()),+ Part::Range(min, max) => KeyRangePart::Range { min, max },+ Part::Bitmap(bitmap) => KeyRangePart::Bitmap(bitmap),+ })+ .collect::<Vec<_>>(),+ );+ // Reference: one bitmap built straight from every key any part may hold.+ let exact = KeyRangeBitmap::try_new(&keys_array(&all_keys), &min, &max, 0);++ // Never excludes a key any part may hold.+ if let Some(union) = &union {+ for key in &all_keys {+ assert!(+ union.may_contain_range(*key as u64, *key as u64),+ "{context}: dropped key {key}"+ );+ }+ }+ if has_bitmap {+ // A part's bitmap can only widen buckets: every bucket the exact+ // bitmap sets, the union sets too.+ match (&union, &exact) {+ (_, None) => assert!(union.is_none(), "{context}"),+ (None, Some(_)) => {}+ (Some(union), Some(exact)) => {+ assert_eq!(+ (union.offset(), union.span(), union.shift()),+ (exact.offset(), exact.span(), exact.shift()),+ "{context}"+ );+ for bucket in exact.bits().set_indices() {+ assert!(union.bits().value(bucket), "{context}: bucket {bucket}");+ }+ }+ }+ // Each set bucket of a part's bitmap stands for its whole key range,+ // so the union equals the one over those ranges given as `Range`s.+ let ranges: Vec<(ScalarValue, ScalarValue)> = parts+ .iter()+ .flat_map(|part| match part {+ Part::Bitmap(bitmap) => bitmap+ .bits()+ .set_indices()+ .map(|bucket| {+ let lo = (bucket as u64) << bitmap.shift();+ let hi = lo+ .saturating_add((1u64 << bitmap.shift()) - 1)+ .min(bitmap.span());+ let key =+ |at: u64| scalar(bitmap.offset().wrapping_add(at) as i64);+ (key(lo), key(hi))+ })+ .collect(),+ _ => vec![],+ })+ .collect();+ let expanded = KeyRangeBitmap::union(+ &min,+ &max,+ &parts+ .iter()+ .filter_map(|part| match part {+ Part::Keys(keys) => Some(KeyRangePart::Keys(keys.as_ref())),+ Part::Range(min, max) => Some(KeyRangePart::Range { min, max }),+ Part::Bitmap(_) => None,+ })+ .chain(+ ranges+ .iter()+ .map(|(min, max)| KeyRangePart::Range { min, max }),+ )+ .collect::<Vec<_>>(),+ );+ assert_eq!(+ union.as_ref().map(parts_of),+ expanded.as_ref().map(parts_of),+ "{context}: bitmap parts differ from their bucket ranges"+ );+ } else {+ // Keys and ranges mark exactly the buckets their keys fall in.+ assert_eq!(+ union.as_ref().map(parts_of),+ exact.as_ref().map(parts_of),+ "{context}"+ );+ }++ std::future::ready(Ok(()))+ }+}diff --git a/datafusion/fuzz/tests/suites/mod.rs b/datafusion/fuzz/tests/suites/mod.rsindex a443d433bd..5edf91e5d6 100644--- a/datafusion/fuzz/tests/suites/mod.rs+++ b/datafusion/fuzz/tests/suites/mod.rs@@ -4,4 +4,5 @@//! Implementation for fuzz test cases.+pub(crate) mod key_range_bitmap;pub(crate) mod spill_pool;diff --git a/datafusion/physical-plan/src/joins/key_range_bitmap.rs b/datafusion/physical-plan/src/joins/key_range_bitmap.rsindex 47030f5cad..2ad9c3369e 100644--- a/datafusion/physical-plan/src/joins/key_range_bitmap.rs+++ b/datafusion/physical-plan/src/joins/key_range_bitmap.rs@@ -16,7 +16,12 @@ use crate::joins::array_map::ArrayMap;const MAX_BUCKETS: u64 = 1 << 20;/// One build partition's contribution to [`KeyRangeBitmap::union`].-pub(crate) enum KeyRangePart<'a> {+///+/// # Public Only for Internal Use:+/// The `datafusion-fuzz` tests require this, but it is not part of the+/// supported public API.+#[doc(hidden)]+pub enum KeyRangePart<'a> {/// The partition's own bitmap.Bitmap(&'a KeyRangeBitmap),/// The partition's build keys, which must lie within its `[min, max]`.@@ -107,7 +112,12 @@ impl KeyRangeBitmap {/// Coarser than the parts' own bitmaps where its buckets are wider, but/// never clears a bucket a part may hold a key in. `None` when the key/// type has no `u64` ordering, or when nothing can be excluded.- pub(crate) fn union(+ ///+ /// # Public Only for Internal Use:+ /// The `datafusion-fuzz` tests require this, but it is not part of the+ /// supported public API.+ #[doc(hidden)]+ pub fn union(min: &ScalarValue,max: &ScalarValue,parts: &[KeyRangePart<'_>],@@ -252,6 +262,32 @@ impl KeyRangeBitmap {}}+/// # Public Only for Internal Use:+/// The `datafusion-fuzz` tests require these accessors, but they are not part+/// of the supported public API.+#[doc(hidden)]+impl KeyRangeBitmap {+ /// Smallest key, in the `u64` ordering the join's array map uses.+ pub fn offset(&self) -> u64 {+ self.offset+ }++ /// `max - min`: a key is in range iff `key.wrapping_sub(offset) <= span`.+ pub fn span(&self) -> u64 {+ self.span+ }++ /// Each bucket covers `1 << shift` keys.+ pub fn shift(&self) -> u32 {+ self.shift+ }++ /// One bit per bucket.+ pub fn bits(&self) -> &BooleanBuffer {+ &self.bits+ }+}+/// One bit per bucket, set for each bucket holding a non-null key of `keys`.fn bucket_bits(keys: &dyn Array, offset: u64, shift: u32, buckets: u64) -> BooleanBuffer {let mut words = MutableBuffer::new_null(buckets as usize);@@ -425,189 +461,4 @@ mod tests {assert!(may(&s, -1, 1));assert!(!may(&s, 1 << 50, 1 << 51));}-- /// `(offset, span, shift, bits)`, to compare bitmaps by value.- fn parts_of(b: &KeyRangeBitmap) -> (u64, u64, u32, &BooleanBuffer) {- (b.offset, b.span, b.shift, &b.bits)- }-- /// One randomly generated scenario, see [`union_fuzz`].- fn run_union_scenario(seed: u64) {- use rand::rngs::StdRng;- use rand::{Rng, SeedableRng};-- let mut rng = StdRng::seed_from_u64(seed);- // Domain widths from one key per bucket to many keys per bucket, so- // the union's buckets are sometimes exact and sometimes coarse.- let width = [64i64, 1 << 12, 1 << 24, 1 << 40][rng.random_range(0..4)];- let start = rng.random_range(-(1i64 << 50)..(1i64 << 50));- let key = |rng: &mut StdRng| start + rng.random_range(0..width);- let keys_array =- |keys: &[i64]| -> ArrayRef { Arc::new(Int64Array::from(keys.to_vec())) };- let scalar = |key: i64| ScalarValue::Int64(Some(key));-- enum Part {- Keys(ArrayRef),- Range(ScalarValue, ScalarValue),- Bitmap(KeyRangeBitmap),- }- // Each part, and every key any part may hold.- let mut parts = vec![];- let mut all_keys = vec![];- let mut has_bitmap = false;- for _ in 0..rng.random_range(1..5) {- if rng.random_bool(1.0 / 3.0) {- // Narrow, so every key it may hold can be listed.- let lo = key(&mut rng);- let hi = lo + rng.random_range(0..64);- parts.push(Part::Range(scalar(lo), scalar(hi)));- all_keys.extend(lo..=hi);- continue;- }- let keys: Vec<i64> = (0..rng.random_range(1..50))- .map(|_| key(&mut rng))- .collect();- let (lo, hi) = (*keys.iter().min().unwrap(), *keys.iter().max().unwrap());- let bitmap = rng- .random_bool(0.5)- .then(|| {- KeyRangeBitmap::try_new(- &keys_array(&keys),- &scalar(lo),- &scalar(hi),- 0,- )- })- .flatten();- parts.push(match bitmap {- Some(bitmap) => {- has_bitmap = true;- Part::Bitmap(bitmap)- }- None => Part::Keys(keys_array(&keys)),- });- all_keys.extend(keys);- }- let min = ScalarValue::Int64(all_keys.iter().min().copied());- let max = ScalarValue::Int64(all_keys.iter().max().copied());- let context = format!(- "seed {seed}: width {width}, {} parts, {} keys",- parts.len(),- all_keys.len()- );-- let union = KeyRangeBitmap::union(- &min,- &max,- &parts- .iter()- .map(|part| match part {- Part::Keys(keys) => KeyRangePart::Keys(keys.as_ref()),- Part::Range(min, max) => KeyRangePart::Range { min, max },- Part::Bitmap(bitmap) => KeyRangePart::Bitmap(bitmap),- })- .collect::<Vec<_>>(),- );- // Reference: one bitmap built straight from every key any part may hold.- let exact = KeyRangeBitmap::try_new(&keys_array(&all_keys), &min, &max, 0);-- // Never excludes a key any part may hold.- if let Some(union) = &union {- for key in &all_keys {- assert!(may(union, *key, *key), "{context}: dropped key {key}");- }- }- if has_bitmap {- // A part's bitmap can only widen buckets: every bucket the exact- // bitmap sets, the union sets too.- match (&union, &exact) {- (_, None) => assert!(union.is_none(), "{context}"),- (None, Some(_)) => {}- (Some(union), Some(exact)) => {- assert_eq!(- (union.offset, union.span, union.shift),- (exact.offset, exact.span, exact.shift),- "{context}"- );- for bucket in exact.bits.set_indices() {- assert!(union.bits.value(bucket), "{context}: bucket {bucket}");- }- }- }- // Each set bucket of a part's bitmap stands for its whole key range,- // so the union equals the one over those ranges given as `Range`s.- let ranges: Vec<(ScalarValue, ScalarValue)> = parts- .iter()- .flat_map(|part| match part {- Part::Bitmap(bitmap) => bitmap- .bits- .set_indices()- .map(|bucket| {- let lo = (bucket as u64) << bitmap.shift;- let hi = lo- .saturating_add((1u64 << bitmap.shift) - 1)- .min(bitmap.span);- let key =- |at: u64| scalar(bitmap.offset.wrapping_add(at) as i64);- (key(lo), key(hi))- })- .collect(),- _ => vec![],- })- .collect();- let expanded = KeyRangeBitmap::union(- &min,- &max,- &parts- .iter()- .filter_map(|part| match part {- Part::Keys(keys) => Some(KeyRangePart::Keys(keys.as_ref())),- Part::Range(min, max) => Some(KeyRangePart::Range { min, max }),- Part::Bitmap(_) => None,- })- .chain(- ranges- .iter()- .map(|(min, max)| KeyRangePart::Range { min, max }),- )- .collect::<Vec<_>>(),- );- assert_eq!(- union.as_ref().map(parts_of),- expanded.as_ref().map(parts_of),- "{context}: bitmap parts differ from their bucket ranges"- );- } else {- // Keys and ranges mark exactly the buckets their keys fall in.- assert_eq!(- union.as_ref().map(parts_of),- exact.as_ref().map(parts_of),- "{context}"- );- }- }-- /// Randomized partitions against [`KeyRangeBitmap::union`]'s contract: it- /// never excludes a key any part may hold, equals the bitmap built from all- /// parts' keys when no part brings its own bitmap, and only widens that- /// bitmap's buckets when one does.- ///- /// `DATAFUSION_KEY_RANGE_UNION_FUZZ_ITERATIONS` and- /// `DATAFUSION_KEY_RANGE_UNION_FUZZ_SEED` select how many scenarios run and- /// from which seed; a failure names its seed so it can be replayed.- #[test]- fn union_fuzz() {- let env_u64 = |name: &str, default: u64| {- std::env::var(name)- .ok()- .and_then(|v| v.parse().ok())- .unwrap_or(default)- };- let first_seed = env_u64("DATAFUSION_KEY_RANGE_UNION_FUZZ_SEED", 0);- let iterations = env_u64("DATAFUSION_KEY_RANGE_UNION_FUZZ_ITERATIONS", 256);- // `saturating_add` so a replay seed near `u64::MAX` cannot overflow.- for seed in first_seed..first_seed.saturating_add(iterations) {- run_union_scenario(seed);- }- }}diff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rsindex a4cfc8f831..e663688310 100644--- a/xtask/src/ci_steps.rs+++ b/xtask/src/ci_steps.rs@@ -16,10 +16,8 @@ const CI_COMMAND: &str = "cargo xtask ci step";const CI_SHORTCUT: &str = "cargo ci-step";const TPCH_DATA_DIR: &str = "datafusion/sqllogictest/test_files/tpch/data";/// Physical-plan fuzzers awaiting migration to the centralized driver.-const LEGACY_FUZZ_TESTS: &[&str] = &[- "joins::hash_join::partitioned_hash_eval::tests::routed_lookup_fuzz",- "joins::key_range_bitmap::tests::union_fuzz",-];+const LEGACY_FUZZ_TESTS: &[&str] =+ &["joins::hash_join::partitioned_hash_eval::tests::routed_lookup_fuzz"];/// Metadata and implementation for one CI step.///@@ -498,8 +496,7 @@ impl StepContext {]).args(LEGACY_FUZZ_TESTS).env("RUST_BACKTRACE", "1")- .env("DATAFUSION_ROUTED_LOOKUP_FUZZ_ITERATIONS", "10000")- .env("DATAFUSION_KEY_RANGE_UNION_FUZZ_ITERATIONS", "10000");+ .env("DATAFUSION_ROUTED_LOOKUP_FUZZ_ITERATIONS", "10000");}"hash-collisions" => {cmd_builder.doc(@@ -990,14 +987,13 @@ cargo test \# Run unmigrated physical-plan fuzz tests with more iterations.cd /workspace && \-DATAFUSION_KEY_RANGE_UNION_FUZZ_ITERATIONS=10000 \DATAFUSION_ROUTED_LOOKUP_FUZZ_ITERATIONS=10000 \RUST_BACKTRACE=1 \cargo test \--profile ci \-p datafusion-physical-plan \--lib \--- --exact joins::hash_join::partitioned_hash_eval::tests::routed_lookup_fuzz joins::key_range_bitmap::tests::union_fuzz+-- --exact joins::hash_join::partitioned_hash_eval::tests::routed_lookup_fuzz")},},