Skip to content

Apache DataFusion SQL Query Engine

7.5 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

//! Shared execution and replay for [`FuzzTest`] experiments.

use std::future::Future;
use std::time::Duration;

use datafusion_common::instant::Instant;
use datafusion_common::{Result, exec_err};

/// Implement this trait to add a fuzzing unit test to DataFusion.
///
/// The fuzz implementation prepares the data and checks one case. The [centralized driver]
/// repeats cases, manages the time budget, and reports failures.
///
/// See the [centralized driver] for how to use it. At a high level:
/// 1. Generate one dataset with `prepare_dataset()`.
/// 2. Run several cases on that dataset with `run_case()`.
/// 3. Repeat steps 1 and 2 until the time budget is reached.
///
/// (All configuration, such as the total time budget, lives in [`RunConfig`].)
///
/// [centralized driver]: https://github.com/apache/datafusion/blob/main/datafusion/fuzz/tests/fuzz.rs
#[expect(async_fn_in_trait)] // Serial execution deliberately does not require Send futures.
pub trait FuzzTest {
    /// Reusable fixtures; `()` is valid when no preparation can be shared.
    type Dataset;

    /// Test identity, including any static variant needed for replay.
    fn name(&self) -> &str;

    /// Build all fixtures from this seed and the test's fixed configuration.
    async fn prepare_dataset(&self, dataset_seed: u64) -> Result<Self::Dataset>;

    /// Execute one complete, independent validation and clean up its resources.
    ///
    /// The implementation should ensure:
    /// - The test case is reproducible given the same dataset and seed.
    /// - If the test fails, the error message includes all the information needed
    ///   to reproduce it (by providing the data seed and case seed).
    async fn run_case(&self, dataset: &Self::Dataset, case_seed: u64) -> Result<()>;

    /// Reproduce the failing case deterministically.
    ///
    /// The input seed should be printed in the error message from [`Self::run_case`]
    /// when the test fails.
    async fn reproduce(&self, dataset_seed: u64, case_seed: u64) -> Result<()> {
        let dataset = with_context(self.name(), None, dataset_seed, None, || {
            self.prepare_dataset(dataset_seed)
        })
        .await?;
        with_context(self.name(), None, dataset_seed, Some(case_seed), || {
            self.run_case(&dataset, case_seed)
        })
        .await
    }
}

/// Execution limits for [`run`]. At least one stopping condition is required.
#[derive(Debug, Clone)]
pub struct RunConfig {
    /// Initial seed, or `None` to choose a fresh seed once per run.
    pub run_seed: Option<u64>,
    /// Maximum successful cases; must be positive when supplied.
    pub max_cases: Option<usize>,
    /// Soft budget including preparation. The first complete case always runs,
    /// even with a zero budget. Active preparation and cases are never cancelled.
    pub time_limit: Option<Duration>,
    /// Maximum cases sharing one fixture; must be positive.
    pub cases_per_dataset: usize,
}

impl RunConfig {
    fn validate(&self) -> Result<()> {
        if self.max_cases.is_none() && self.time_limit.is_none() {
            return exec_err!("Fuzz run requires max_cases or time_limit");
        }
        if self.max_cases == Some(0) {
            return exec_err!("Fuzz run max_cases must be greater than zero");
        }
        if self.cases_per_dataset == 0 {
            return exec_err!("Fuzz run cases_per_dataset must be greater than zero");
        }
        Ok(())
    }

    // Keep admission separate from the clock so deadline boundaries can be
    // tested without sleeping. Only the first case may start past the deadline.
    fn should_stop(&self, cases_passed: usize, elapsed: Duration) -> bool {
        self.max_cases.is_some_and(|limit| cases_passed >= limit)
            || (cases_passed > 0 && self.time_limit.is_some_and(|limit| elapsed >= limit))
    }
}

/// Counters returned by a successful campaign. Failures stop the run immediately.
#[derive(Debug)]
pub struct RunStats {
    /// Resolved seed, including when the caller requested a fresh one.
    pub run_seed: u64,
    /// Number of successfully prepared fixtures.
    pub datasets_prepared: usize,
    /// Number of complete validations that returned successfully.
    pub cases_passed: usize,
    /// Total execution time, including fixture preparation and cleanup.
    pub elapsed: Duration,
}

/// Run cases serially until either limit is reached or a failure occurs.
///
/// Case seeds are `run_seed + case_index`, wrapping at `u64::MAX`. Dataset seeds
/// independently use `run_seed + dataset_index`. Neither sequence depends on
/// randomness consumed by the test. This also lets existing consecutive-seed
/// fuzzers retain their coverage. A timed run may finish different case counts
/// on different machines; use a fixed count for a repeatable seed sequence.
pub async fn run<T: FuzzTest>(test: &T, config: RunConfig) -> Result<RunStats> {
    config.validate()?;
    let run_seed = config.run_seed.unwrap_or_else(rand::random);
    let start = Instant::now();
    let mut stats = RunStats {
        run_seed,
        datasets_prepared: 0,
        cases_passed: 0,
        elapsed: Duration::ZERO,
    };
    while !config.should_stop(stats.cases_passed, start.elapsed()) {
        let dataset_seed = run_seed.wrapping_add(stats.datasets_prepared as u64);
        let dataset =
            with_context(test.name(), Some(run_seed), dataset_seed, None, || {
                test.prepare_dataset(dataset_seed)
            })
            .await?;
        stats.datasets_prepared += 1;
        for _ in 0..config.cases_per_dataset {
            if config.should_stop(stats.cases_passed, start.elapsed()) {
                break;
            }
            let case_seed = run_seed.wrapping_add(stats.cases_passed as u64);
            with_context(
                test.name(),
                Some(run_seed),
                dataset_seed,
                Some(case_seed),
                || test.run_case(&dataset, case_seed),
            )
            .await?;
            stats.cases_passed += 1;
        }
    }
    stats.elapsed = start.elapsed();
    Ok(stats)
}

/// Print before invoking the operation so libtest captures context even for
/// assertion panics. Returned errors retain their original source; panics unwind
/// normally with their original payload and hook, without process-global changes.
async fn with_context<T, F: Future<Output = Result<T>>>(
    name: &str,
    run_seed: Option<u64>,
    dataset_seed: u64,
    case_seed: Option<u64>,
    operation: impl FnOnce() -> F,
) -> Result<T> {
    let phase = if case_seed.is_some() {
        "run_case"
    } else {
        "prepare_dataset"
    };
    let run_context = run_seed
        .map(|seed| format!("Run seed: {seed}\n"))
        .unwrap_or_default();
    let instruction = if let Some(case_seed) = case_seed {
        format!(
            "Case seed: {case_seed}\nReproduce with this test's configuration: \
             test.reproduce({dataset_seed}, {case_seed}).await"
        )
    } else {
        format!(
            "Retry with this test's configuration: \
             test.prepare_dataset({dataset_seed}).await"
        )
    };
    let context = format!(
        "Fuzz test: {name}\nPhase: {phase}\n{run_context}Dataset seed: {dataset_seed}\n{instruction}"
    );
    eprintln!("{context}");
    operation().await.map_err(|error| error.context(context))
}