Skip to content

Apache DataFusion SQL Query Engine

4.0 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

//! See `main.rs` for how to run it.

use std::sync::Arc;

use datafusion::datasource::file_format::parquet::ParquetFormat;
use datafusion::datasource::listing::ListingOptions;
use datafusion::datasource::physical_plan::{FileGroup, ParquetSource};
use datafusion::datasource::source::DataSourceExec;
use datafusion::error::DataFusionError;
use datafusion::execution::context::SessionContext;
use datafusion::physical_plan::metrics::MetricValue;
use datafusion::physical_plan::{
    ExecutionPlan, ExecutionPlanVisitor, execute_stream, visit_execution_plan,
};
use datafusion_examples::utils::{datasets::ExampleDataset, write_csv_to_parquet};
use futures::StreamExt;

/// Example of collecting metrics after execution by visiting the `ExecutionPlan`
pub async fn parquet_exec_visitor() -> datafusion::common::Result<()> {
    let ctx = SessionContext::new();

    // Convert the CSV input into a temporary Parquet directory for querying
    let dataset = ExampleDataset::Cars;
    let parquet_temp = write_csv_to_parquet(&ctx, &dataset.path()).await?;

    // Configure listing options
    let file_format = ParquetFormat::default().with_enable_pruning(true);
    let listing_options = ListingOptions::new(Arc::new(file_format));

    let table_path = parquet_temp.file_uri()?;

    // First example were we use an absolute path, which requires no additional setup.
    ctx.register_listing_table(
        "my_table",
        &table_path,
        listing_options.clone(),
        None,
        None,
    )
    .await?;

    let df = ctx.sql("SELECT * FROM my_table").await?;
    let plan = df.create_physical_plan().await?;

    // Create empty visitor
    let mut visitor = ParquetExecVisitor {
        file_groups: None,
        bytes_scanned: None,
    };

    // Make sure you execute the plan to collect actual execution statistics.
    // For example, in this example the `file_scan_config` is known without executing
    // but the `bytes_scanned` would be None if we did not execute.
    let mut batch_stream = execute_stream(plan.clone(), ctx.task_ctx())?;
    while let Some(batch) = batch_stream.next().await {
        println!("Batch rows: {}", batch.unwrap().num_rows());
    }

    visit_execution_plan(plan.as_ref(), &mut visitor)?;

    println!(
        "ParquetExecVisitor bytes_scanned: {:?}",
        visitor.bytes_scanned
    );
    println!(
        "ParquetExecVisitor file_groups: {:?}",
        visitor.file_groups.unwrap()
    );

    Ok(())
}

/// Define a struct with fields to hold the execution information you want to
/// collect. In this case, I want information on how many bytes were scanned
/// and `file_groups` from the FileScanConfig.
#[derive(Debug)]
struct ParquetExecVisitor {
    file_groups: Option<Vec<FileGroup>>,
    bytes_scanned: Option<MetricValue>,
}

impl ExecutionPlanVisitor for ParquetExecVisitor {
    type Error = DataFusionError;

    /// This function is called once for every node in the tree.
    /// Based on your needs implement either `pre_visit` (visit each node before its children/inputs)
    /// or `post_visit` (visit each node after its children/inputs)
    fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result<bool, Self::Error> {
        // If needed match on a specific `ExecutionPlan` node type
        if let Some(data_source_exec) = plan.downcast_ref::<DataSourceExec>()
            && let Some((file_config, _)) =
                data_source_exec.downcast_to_file_source::<ParquetSource>()
        {
            self.file_groups = Some(file_config.file_groups.clone());

            let Some(metrics) = data_source_exec.metrics() else {
                return Ok(true);
            };
            self.bytes_scanned = metrics.sum_by_name("bytes_scanned");
        }
        Ok(true)
    }
}