Skip to content

Apache DataFusion SQL Query Engine

fix: Fix flaky sort fuzzer test (#26983)

Yongting Youce0e8293f905
diff --git a/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs b/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs
index b3aec96b23..a8ad491808 100644
--- a/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs
+++ b/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs
@@ -10,6 +10,7 @@ use std::sync::Arc;
use arrow::array::RecordBatch;
use arrow::compute::concat_batches;
+use arrow::util::pretty::pretty_format_batches;
use arrow_schema::SchemaRef;
use datafusion::datasource::MemTable;
use datafusion::prelude::{SessionConfig, SessionContext};
@@ -210,9 +211,7 @@ impl SortQueryFuzzer {
if expected_results.is_none() {
expected_results = Some(results);
} else if let Some(ref expected) = expected_results {
- // Every projected column is an ORDER BY key, so tied
- // rows have identical output and can be compared in order.
- assert_eq!(expected, &results);
+ assert_batches_equal_in_order(expected, &results, "");
} else {
unreachable!();
}
@@ -401,6 +400,9 @@ impl SortFuzzerTestGenerator {
/// Generates a random SQL query string and an optional limit value.
/// Returns a tuple containing the query string and an optional limit.
+ ///
+ /// Every projected column is an ORDER BY key, so tied rows have identical
+ /// output and results can be compared in order, including with LIMIT.
pub fn generate_random_query(&self, rng_seed: u64) -> (String, Option<usize>) {
let mut rng = StdRng::seed_from_u64(rng_seed);
@@ -593,10 +595,10 @@ impl SortFuzzerTestGenerator {
let ctx =
self.generate_random_config(config_seed, false, Some(threshold))?;
let actual = ctx.sql(&query_str).await?.collect().await?;
- assert_eq!(
- results,
- concat_batches(&schema, &actual)?,
- "sort_in_place_threshold_bytes={threshold}"
+ assert_batches_equal_in_order(
+ &results,
+ &concat_batches(&schema, &actual)?,
+ &format!("sort_in_place_threshold_bytes={threshold}"),
);
}
}
@@ -605,10 +607,58 @@ impl SortFuzzerTestGenerator {
}
}
+/// Compares rows in order by their displayed values, so logically equal
+/// batches with different physical encodings (e.g. dictionaries) still match.
+#[track_caller]
+fn assert_batches_equal_in_order(
+ expected: &RecordBatch,
+ actual: &RecordBatch,
+ msg: &str,
+) {
+ let expected = pretty_format_batches(std::slice::from_ref(expected))
+ .unwrap()
+ .to_string();
+ let actual = pretty_format_batches(std::slice::from_ref(actual))
+ .unwrap()
+ .to_string();
+ assert_eq!(expected, actual, "{msg}");
+}
+
#[cfg(test)]
mod test {
use super::*;
+ /// The sort query fuzzer uses the current time as its seed for each run.
+ /// This test replays seeds from previously failing cases.
+ ///
+ /// Regressions:
+ /// https://github.com/apache/datafusion/actions/runs/41196052207/job/124522323666
+ #[tokio::test(flavor = "multi_thread")]
+ async fn test_regression_sort_query_fuzz() {
+ // Regression seeds
+ let gen_seed = 1_793_445_468;
+
+ // Reproducing
+ let mut test_generator = SortFuzzerTestGenerator::new(
+ 2000,
+ 3,
+ "sort_fuzz_table".to_string(),
+ get_supported_types_columns(gen_seed),
+ false,
+ gen_seed,
+ );
+
+ test_generator
+ .fuzzer_run(
+ 17_320_197_313_368_524_478,
+ 16_774_755_103_655_697_964,
+ 1_568_822_825_954_225_201,
+ true,
+ )
+ .await
+ .unwrap();
+ }
+
/// Given the same seed, the result should be the same
#[tokio::test]
async fn test_sort_query_fuzzer_deterministic() {
@@ -624,6 +674,6 @@ mod test {
let res1 = test_generator.fuzzer_run(1, 2, 3, false).await.unwrap();
let res2 = test_generator.fuzzer_run(1, 2, 3, false).await.unwrap();
- assert_eq!(res1, res2);
+ assert_batches_equal_in_order(&res1, &res2, "");
}
}