Apache DataFusion SQL Query Engine
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
//! See `main.rs` for how to run it.
use std::sync::Arc;
use datafusion::datasource::file_format::parquet::ParquetFormat;
use datafusion::datasource::listing::ListingOptions;
use datafusion::datasource::physical_plan::{FileGroup, ParquetSource};
use datafusion::datasource::source::DataSourceExec;
use datafusion::error::DataFusionError;
use datafusion::execution::context::SessionContext;
use datafusion::physical_plan::metrics::MetricValue;
use datafusion::physical_plan::{
ExecutionPlan, ExecutionPlanVisitor, execute_stream, visit_execution_plan,
};
use datafusion_examples::utils::{datasets::ExampleDataset, write_csv_to_parquet};
use futures::StreamExt;
/// Example of collecting metrics after execution by visiting the `ExecutionPlan`
pub async fn parquet_exec_visitor() -> datafusion::common::Result<()> {
let ctx = SessionContext::new();
// Convert the CSV input into a temporary Parquet directory for querying
let dataset = ExampleDataset::Cars;
let parquet_temp = write_csv_to_parquet(&ctx, &dataset.path()).await?;
// Configure listing options
let file_format = ParquetFormat::default().with_enable_pruning(true);
let listing_options = ListingOptions::new(Arc::new(file_format));
let table_path = parquet_temp.file_uri()?;
// First example were we use an absolute path, which requires no additional setup.
ctx.register_listing_table(
"my_table",
&table_path,
listing_options.clone(),
None,
None,
)
.await?;
let df = ctx.sql("SELECT * FROM my_table").await?;
let plan = df.create_physical_plan().await?;
// Create empty visitor
let mut visitor = ParquetExecVisitor {
file_groups: None,
bytes_scanned: None,
};
// Make sure you execute the plan to collect actual execution statistics.
// For example, in this example the `file_scan_config` is known without executing
// but the `bytes_scanned` would be None if we did not execute.
let mut batch_stream = execute_stream(plan.clone(), ctx.task_ctx())?;
while let Some(batch) = batch_stream.next().await {
println!("Batch rows: {}", batch.unwrap().num_rows());
}
visit_execution_plan(plan.as_ref(), &mut visitor)?;
println!(
"ParquetExecVisitor bytes_scanned: {:?}",
visitor.bytes_scanned
);
println!(
"ParquetExecVisitor file_groups: {:?}",
visitor.file_groups.unwrap()
);
Ok(())
}
/// Define a struct with fields to hold the execution information you want to
/// collect. In this case, I want information on how many bytes were scanned
/// and `file_groups` from the FileScanConfig.
#[derive(Debug)]
struct ParquetExecVisitor {
file_groups: Option<Vec<FileGroup>>,
bytes_scanned: Option<MetricValue>,
}
impl ExecutionPlanVisitor for ParquetExecVisitor {
type Error = DataFusionError;
/// This function is called once for every node in the tree.
/// Based on your needs implement either `pre_visit` (visit each node before its children/inputs)
/// or `post_visit` (visit each node after its children/inputs)
fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result<bool, Self::Error> {
// If needed match on a specific `ExecutionPlan` node type
if let Some(data_source_exec) = plan.downcast_ref::<DataSourceExec>()
&& let Some((file_config, _)) =
data_source_exec.downcast_to_file_source::<ParquetSource>()
{
self.file_groups = Some(file_config.file_groups.clone());
let Some(metrics) = data_source_exec.metrics() else {
return Ok(true);
};
self.bytes_scanned = metrics.sum_by_name("bytes_scanned");
}
Ok(true)
}
}