Apache DataFusion SQL Query Engine
feat: allow custom caching via logical node (#18688)
yqrz6ea305ec4c15
diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex 08ca1a176e..8e52ca1bf1 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -2330,6 +2330,10 @@ impl DataFrame {/// Cache DataFrame as a memory table.///+ /// Default behavior could be changed using+ /// a [`crate::execution::session_state::CacheFactory`]+ /// configured via [`SessionState`].+ ////// ```/// # use datafusion::prelude::*;/// # use datafusion::error::Result;@@ -2344,14 +2348,20 @@ impl DataFrame {/// # }/// ```pub async fn cache(self) -> Result<DataFrame> {- let context = SessionContext::new_with_state((*self.session_state).clone());- // The schema is consistent with the output- let plan = self.clone().create_physical_plan().await?;- let schema = plan.schema();- let task_ctx = Arc::new(self.task_ctx());- let partitions = collect_partitioned(plan, task_ctx).await?;- let mem_table = MemTable::try_new(schema, partitions)?;- context.read_table(Arc::new(mem_table))+ if let Some(cache_factory) = self.session_state.cache_factory() {+ let new_plan =+ cache_factory.create(self.plan, self.session_state.as_ref())?;+ Ok(Self::new(*self.session_state, new_plan))+ } else {+ let context = SessionContext::new_with_state((*self.session_state).clone());+ // The schema is consistent with the output+ let plan = self.clone().create_physical_plan().await?;+ let schema = plan.schema();+ let task_ctx = Arc::new(self.task_ctx());+ let partitions = collect_partitioned(plan, task_ctx).await?;+ let mem_table = MemTable::try_new(schema, partitions)?;+ context.read_table(Arc::new(mem_table))+ }}/// Apply an alias to the DataFrame.diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rsindex d7a66db28a..8bc526fbfb 100644--- a/datafusion/core/src/execution/session_state.rs+++ b/datafusion/core/src/execution/session_state.rs@@ -185,6 +185,7 @@ pub struct SessionState {/// It will be invoked on `CREATE FUNCTION` statements./// thus, changing dialect o PostgreSql is requiredfunction_factory: Option<Arc<dyn FunctionFactory>>,+ cache_factory: Option<Arc<dyn CacheFactory>>,/// Cache logical plans of prepared statements for later execution./// Key is the prepared statement name.prepared_plans: HashMap<String, Arc<PreparedPlan>>,@@ -206,6 +207,7 @@ impl Debug for SessionState {.field("table_options", &self.table_options).field("table_factories", &self.table_factories).field("function_factory", &self.function_factory)+ .field("cache_factory", &self.cache_factory).field("expr_planners", &self.expr_planners);#[cfg(feature = "sql")]@@ -355,6 +357,16 @@ impl SessionState {self.function_factory.as_ref()}+ /// Register a [`CacheFactory`] for custom caching strategy+ pub fn set_cache_factory(&mut self, cache_factory: Arc<dyn CacheFactory>) {+ self.cache_factory = Some(cache_factory);+ }++ /// Get the cache factory+ pub fn cache_factory(&self) -> Option<&Arc<dyn CacheFactory>> {+ self.cache_factory.as_ref()+ }+/// Get the table factoriespub fn table_factories(&self) -> &HashMap<String, Arc<dyn TableProviderFactory>> {&self.table_factories@@ -941,6 +953,7 @@ pub struct SessionStateBuilder {table_factories: Option<HashMap<String, Arc<dyn TableProviderFactory>>>,runtime_env: Option<Arc<RuntimeEnv>>,function_factory: Option<Arc<dyn FunctionFactory>>,+ cache_factory: Option<Arc<dyn CacheFactory>>,// fields to support convenience functionsanalyzer_rules: Option<Vec<Arc<dyn AnalyzerRule + Send + Sync>>>,optimizer_rules: Option<Vec<Arc<dyn OptimizerRule + Send + Sync>>>,@@ -978,6 +991,7 @@ impl SessionStateBuilder {table_factories: None,runtime_env: None,function_factory: None,+ cache_factory: None,// fields to support convenience functionsanalyzer_rules: None,optimizer_rules: None,@@ -1030,7 +1044,7 @@ impl SessionStateBuilder {table_factories: Some(existing.table_factories),runtime_env: Some(existing.runtime_env),function_factory: existing.function_factory,-+ cache_factory: existing.cache_factory,// fields to support convenience functionsanalyzer_rules: None,optimizer_rules: None,@@ -1319,6 +1333,15 @@ impl SessionStateBuilder {self}+ /// Set a [`CacheFactory`] for custom caching strategy+ pub fn with_cache_factory(+ mut self,+ cache_factory: Option<Arc<dyn CacheFactory>>,+ ) -> Self {+ self.cache_factory = cache_factory;+ self+ }+/// Register an `ObjectStore` to the [`RuntimeEnv`]. See [`RuntimeEnv::register_object_store`]/// for more details.///@@ -1382,6 +1405,7 @@ impl SessionStateBuilder {table_factories,runtime_env,function_factory,+ cache_factory,analyzer_rules,optimizer_rules,physical_optimizer_rules,@@ -1418,6 +1442,7 @@ impl SessionStateBuilder {table_factories: table_factories.unwrap_or_default(),runtime_env,function_factory,+ cache_factory,prepared_plans: HashMap::new(),};@@ -1621,6 +1646,11 @@ impl SessionStateBuilder {&mut self.function_factory}+ /// Returns the cache factory+ pub fn cache_factory(&mut self) -> &mut Option<Arc<dyn CacheFactory>> {+ &mut self.cache_factory+ }+/// Returns the current analyzer_rules valuepub fn analyzer_rules(&mut self,@@ -1659,6 +1689,7 @@ impl Debug for SessionStateBuilder {.field("table_options", &self.table_options).field("table_factories", &self.table_factories).field("function_factory", &self.function_factory)+ .field("cache_factory", &self.cache_factory).field("expr_planners", &self.expr_planners);#[cfg(feature = "sql")]let ret = ret.field("type_planner", &self.type_planner);@@ -2047,6 +2078,19 @@ pub(crate) struct PreparedPlan {pub(crate) plan: Arc<LogicalPlan>,}+/// A [`CacheFactory`] can be registered via [`SessionState`]+/// to create a custom logical plan for [`crate::dataframe::DataFrame::cache`].+/// Additionally, a custom [`crate::physical_planner::ExtensionPlanner`]/[`QueryPlanner`]+/// may need to be implemented to handle such plans.+pub trait CacheFactory: Debug + Send + Sync {+ /// Create a logical plan for caching+ fn create(+ &self,+ plan: LogicalPlan,+ session_state: &SessionState,+ ) -> datafusion_common::Result<LogicalPlan>;+}+#[cfg(test)]mod tests {use super::{SessionContextProvider, SessionStateBuilder};diff --git a/datafusion/core/src/test_util/mod.rs b/datafusion/core/src/test_util/mod.rsindex 7149c5b0bd..466ee38a42 100644--- a/datafusion/core/src/test_util/mod.rs+++ b/datafusion/core/src/test_util/mod.rs@@ -25,6 +25,7 @@ pub mod csv;use futures::Stream;use std::any::Any;use std::collections::HashMap;+use std::fmt::Formatter;use std::fs::File;use std::io::Write;use std::path::Path;@@ -36,16 +37,20 @@ use crate::dataframe::DataFrame;use crate::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable};use crate::datasource::{empty::EmptyTable, provider_as_source};use crate::error::Result;+use crate::execution::session_state::CacheFactory;use crate::logical_expr::{LogicalPlanBuilder, UNNAMED_TABLE};use crate::physical_plan::ExecutionPlan;use crate::prelude::{CsvReadOptions, SessionContext};-use crate::execution::SendableRecordBatchStream;+use crate::execution::{SendableRecordBatchStream, SessionState, SessionStateBuilder};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_catalog::Session;-use datafusion_common::TableReference;-use datafusion_expr::{CreateExternalTable, Expr, SortExpr, TableType};+use datafusion_common::{DFSchemaRef, TableReference};+use datafusion_expr::{+ CreateExternalTable, Expr, LogicalPlan, SortExpr, TableType,+ UserDefinedLogicalNodeCore,+};use std::pin::Pin;use async_trait::async_trait;@@ -282,3 +287,67 @@ impl RecordBatchStream for BoundedStream {self.record_batch.schema()}}++#[derive(Hash, Eq, PartialEq, PartialOrd, Debug)]+struct CacheNode {+ input: LogicalPlan,+}++impl UserDefinedLogicalNodeCore for CacheNode {+ fn name(&self) -> &str {+ "CacheNode"+ }++ fn inputs(&self) -> Vec<&LogicalPlan> {+ vec![&self.input]+ }++ fn schema(&self) -> &DFSchemaRef {+ self.input.schema()+ }++ fn expressions(&self) -> Vec<Expr> {+ vec![]+ }++ fn fmt_for_explain(&self, f: &mut Formatter) -> std::fmt::Result {+ write!(f, "CacheNode")+ }++ fn with_exprs_and_inputs(+ &self,+ _exprs: Vec<Expr>,+ inputs: Vec<LogicalPlan>,+ ) -> Result<Self> {+ assert_eq!(inputs.len(), 1, "input size inconsistent");+ Ok(Self {+ input: inputs[0].clone(),+ })+ }+}++#[derive(Debug)]+struct TestCacheFactory {}++impl CacheFactory for TestCacheFactory {+ fn create(+ &self,+ plan: LogicalPlan,+ _session_state: &SessionState,+ ) -> Result<LogicalPlan> {+ Ok(LogicalPlan::Extension(datafusion_expr::Extension {+ node: Arc::new(CacheNode { input: plan }),+ }))+ }+}++/// Create a test table registered to a session context with an associated cache factory+pub async fn test_table_with_cache_factory() -> Result<DataFrame> {+ let session_state = SessionStateBuilder::new()+ .with_cache_factory(Some(Arc::new(TestCacheFactory {})))+ .build();+ let ctx = SessionContext::new_with_state(session_state);+ let name = "aggregate_test_100";+ register_aggregate_csv(&ctx, name).await?;+ ctx.table(name).await+}diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex b856a776c8..fb6dc3bcba 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -61,7 +61,7 @@ use datafusion::prelude::{};use datafusion::test_util::{parquet_test_data, populate_csv_partitions, register_aggregate_csv, test_table,- test_table_with_name,+ test_table_with_cache_factory, test_table_with_name,};use datafusion_catalog::TableProvider;use datafusion_common::test_util::{batches_to_sort_string, batches_to_string};@@ -2335,6 +2335,29 @@ async fn cache_test() -> Result<()> {Ok(())}+#[tokio::test]+async fn cache_producer_test() -> Result<()> {+ let df = test_table_with_cache_factory()+ .await?+ .select_columns(&["c2", "c3"])?+ .limit(0, Some(1))?+ .with_column("sum", cast(col("c2") + col("c3"), DataType::Int64))?;++ let cached_df = df.clone().cache().await?;++ assert_snapshot!(+ cached_df.clone().into_optimized_plan().unwrap(),+ @r###"+ CacheNode+ Projection: aggregate_test_100.c2, aggregate_test_100.c3, CAST(CAST(aggregate_test_100.c2 AS Int64) + CAST(aggregate_test_100.c3 AS Int64) AS Int64) AS sum+ Projection: aggregate_test_100.c2, aggregate_test_100.c3+ Limit: skip=0, fetch=1+ TableScan: aggregate_test_100, fetch=1+ "###+ );+ Ok(())+}+#[tokio::test]async fn partition_aware_union() -> Result<()> {let left = test_table().await?.select_columns(&["c1", "c2"])?;