Apache DataFusion SQL Query Engine
Move generate_series projection logic into LazyMemoryStream (#18373)
Michael Kleenc5fb60535ea2
diff --git a/datafusion/functions-table/src/generate_series.rs b/datafusion/functions-table/src/generate_series.rsindex c66e652147..d71c5945aa 100644--- a/datafusion/functions-table/src/generate_series.rs+++ b/datafusion/functions-table/src/generate_series.rs@@ -237,7 +237,6 @@ impl GenerateSeriesTable {pub fn as_generator(&self,batch_size: usize,- projection: Option<Vec<usize>>,) -> Result<Arc<RwLock<dyn LazyBatchGenerator>>> {let generator: Arc<RwLock<dyn LazyBatchGenerator>> = match &self.args {GenSeriesArgs::ContainsNull { name } => Arc::new(RwLock::new(Empty { name })),@@ -256,7 +255,6 @@ impl GenerateSeriesTable {batch_size,include_end: *include_end,name,- projection,})),GenSeriesArgs::TimestampArgs {start,@@ -297,7 +295,6 @@ impl GenerateSeriesTable {batch_size,include_end: *include_end,name,- projection,}))}GenSeriesArgs::DateArgs {@@ -327,7 +324,6 @@ impl GenerateSeriesTable {batch_size,include_end: *include_end,name,- projection,})),};@@ -345,7 +341,6 @@ pub struct GenericSeriesState<T: SeriesValue> {current: T,include_end: bool,name: &'static str,- projection: Option<Vec<usize>>,}impl<T: SeriesValue> GenericSeriesState<T> {@@ -401,11 +396,7 @@ impl<T: SeriesValue> LazyBatchGenerator for GenericSeriesState<T> {let array = self.current.create_array(buf)?;let batch = RecordBatch::try_new(Arc::clone(&self.schema), vec![array])?;- let projected = match self.projection.as_ref() {- Some(projection) => batch.project(projection)?,- None => batch,- };- Ok(Some(projected))+ Ok(Some(batch))}}@@ -481,14 +472,12 @@ impl TableProvider for GenerateSeriesTable {_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {let batch_size = state.config_options().execution.batch_size;- let schema = match projection {- Some(projection) => Arc::new(self.schema.project(projection)?),- None => self.schema(),- };-- let generator = self.as_generator(batch_size, projection.cloned())?;+ let generator = self.as_generator(batch_size)?;- Ok(Arc::new(LazyMemoryExec::try_new(schema, vec![generator])?))+ Ok(Arc::new(+ LazyMemoryExec::try_new(self.schema(), vec![generator])?+ .with_projection(projection.cloned()),+ ))}}diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rsindex 1bf1e04efb..09710ae1e2 100644--- a/datafusion/physical-plan/src/memory.rs+++ b/datafusion/physical-plan/src/memory.rs@@ -153,6 +153,8 @@ pub trait LazyBatchGenerator: Send + Sync + fmt::Debug + fmt::Display {pub struct LazyMemoryExec {/// Schema representing the dataschema: SchemaRef,+ /// Optional projection for which columns to load+ projection: Option<Vec<usize>>,/// Functions to generate batches for each partitionbatch_generators: Vec<Arc<RwLock<dyn LazyBatchGenerator>>>,/// Plan properties cache storing equivalence properties, partitioning, and execution mode@@ -199,12 +201,28 @@ impl LazyMemoryExec {Ok(Self {schema,+ projection: None,batch_generators: generators,cache,metrics: ExecutionPlanMetricsSet::new(),})}+ pub fn with_projection(mut self, projection: Option<Vec<usize>>) -> Self {+ match projection.as_ref() {+ Some(columns) => {+ let projected = Arc::new(self.schema.project(columns).unwrap());+ self.cache = self.cache.with_eq_properties(EquivalenceProperties::new(+ Arc::clone(&projected),+ ));+ self.schema = projected;+ self.projection = projection;+ self+ }+ _ => self,+ }+ }+pub fn try_set_partitioning(&mut self, partitioning: Partitioning) -> Result<()> {if partitioning.partition_count() != self.batch_generators.len() {internal_err!(@@ -320,6 +338,7 @@ impl ExecutionPlan for LazyMemoryExec {let stream = LazyMemoryStream {schema: Arc::clone(&self.schema),+ projection: self.projection.clone(),generator: Arc::clone(&self.batch_generators[partition]),baseline_metrics,};@@ -338,6 +357,8 @@ impl ExecutionPlan for LazyMemoryExec {/// Stream that generates record batches on demandpub struct LazyMemoryStream {schema: SchemaRef,+ /// Optional projection for which columns to load+ projection: Option<Vec<usize>>,/// Generator to produce batches////// Note: Idiomatically, DataFusion uses plan-time parallelism - each stream@@ -361,7 +382,14 @@ impl Stream for LazyMemoryStream {let batch = self.generator.write().generate_next_batch();let poll = match batch {- Ok(Some(batch)) => Poll::Ready(Some(Ok(batch))),+ Ok(Some(batch)) => {+ // return just the columns requested+ let batch = match self.projection.as_ref() {+ Some(columns) => batch.project(columns)?,+ None => batch,+ };+ Poll::Ready(Some(Ok(batch)))+ }Ok(None) => Poll::Ready(None),Err(e) => Poll::Ready(Some(Err(e))),};diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex 0ebbb373f2..e5f4a1f7d0 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -1940,8 +1940,7 @@ impl protobuf::PhysicalPlanNode {};let table = GenerateSeriesTable::new(Arc::clone(&schema), args);- let generator =- table.as_generator(generate_series.target_batch_size as usize, None)?;+ let generator = table.as_generator(generate_series.target_batch_size as usize)?;Ok(Arc::new(LazyMemoryExec::try_new(schema, vec![generator])?))}