Apache DataFusion SQL Query Engine
feat: serialize LazyMemoryExec through generator hooks (#25765)
QinXid38c545750f3
diff --git a/Cargo.lock b/Cargo.lockindex e6cb96aeb3..47b2d0fb10 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2374,6 +2374,8 @@ dependencies = ["datafusion-expr","datafusion-physical-expr","datafusion-physical-plan",+ "datafusion-proto-common",+ "datafusion-proto-models","parking_lot",]@@ -2595,6 +2597,7 @@ dependencies = ["doc-comment","flate2","object_store",+ "parking_lot","pretty_assertions","prost","recursive",diff --git a/datafusion/functions-table/Cargo.toml b/datafusion/functions-table/Cargo.tomlindex fb02c2c5e2..11015e5ed5 100644--- a/datafusion/functions-table/Cargo.toml+++ b/datafusion/functions-table/Cargo.toml@@ -40,6 +40,13 @@ workspace = true[lib]name = "datafusion_functions_table"+[features]+proto = [+ "dep:datafusion-proto-common",+ "dep:datafusion-proto-models",+ "datafusion-physical-plan/proto",+]+[dependencies]arrow = { workspace = true }async-trait = { workspace = true }@@ -48,6 +55,8 @@ datafusion-common = { workspace = true }datafusion-expr = { workspace = true }datafusion-physical-expr = { workspace = true }datafusion-physical-plan = { workspace = true }+datafusion-proto-common = { workspace = true, optional = true }+datafusion-proto-models = { workspace = true, optional = true }parking_lot = { workspace = true }[dev-dependencies]diff --git a/datafusion/functions-table/src/generate_series.rs b/datafusion/functions-table/src/generate_series.rsindex 32db9bc5be..89610e56cc 100644--- a/datafusion/functions-table/src/generate_series.rs+++ b/datafusion/functions-table/src/generate_series.rs@@ -39,6 +39,9 @@ use std::fmt;use std::str::FromStr;use std::sync::Arc;+#[cfg(feature = "proto")]+pub mod proto;+/// Empty generator that produces no rows - used when series arguments contain null values#[derive(Debug, Clone)]pub struct Empty {@@ -63,6 +66,23 @@ impl LazyBatchGenerator for Empty {fn reset_state(&self) -> Arc<RwLock<dyn LazyBatchGenerator>> {Arc::new(RwLock::new(Empty { name: self.name }))}++ #[cfg(feature = "proto")]+ fn try_to_proto(+ &self,+ ) -> Result<Option<datafusion_proto_models::protobuf::GenerateSeriesNode>> {+ use datafusion_proto_models::protobuf;+ Ok(Some(protobuf::GenerateSeriesNode {+ schema: None,+ // Preserve the batch size used by the legacy empty generator codec.+ target_batch_size: 8192,+ args: Some(protobuf::generate_series_node::Args::ContainsNull(+ protobuf::GenerateSeriesArgsContainsNull {+ name: proto::encode_name(self.name)? as i32,+ },+ )),+ }))+ }}impl fmt::Display for Empty {@@ -99,6 +119,22 @@ pub trait SeriesValue: fmt::Debug + Clone + Send + Sync + 'static {/// Display the value for debuggingfn display_value(&self) -> String;++ /// Encode the arguments of a series with this value type.+ ///+ /// Implementations without a built-in wire representation return `None`,+ /// allowing the owning plan to use an extension codec instead.+ #[cfg(feature = "proto")]+ fn try_to_proto_args(+ &self,+ _end: &Self,+ _step: &Self::StepType,+ _include_end: bool,+ _name: &str,+ ) -> Result<Option<datafusion_proto_models::protobuf::generate_series_node::Args>>+ {+ Ok(None)+ }}impl SeriesValue for i64 {@@ -141,6 +177,27 @@ impl SeriesValue for i64 {fn display_value(&self) -> String {self.to_string()}++ #[cfg(feature = "proto")]+ fn try_to_proto_args(+ &self,+ end: &Self,+ step: &Self::StepType,+ include_end: bool,+ name: &str,+ ) -> Result<Option<datafusion_proto_models::protobuf::generate_series_node::Args>>+ {+ use datafusion_proto_models::protobuf;+ Ok(Some(protobuf::generate_series_node::Args::Int64Args(+ protobuf::GenerateSeriesArgsInt64 {+ start: *self,+ end: *end,+ step: *step,+ include_end,+ name: proto::encode_name(name)? as i32,+ },+ )))+ }}#[derive(Debug, Clone)]@@ -237,6 +294,46 @@ impl SeriesValue for TimestampValue {fn display_value(&self) -> String {self.value.to_string()}++ #[cfg(feature = "proto")]+ fn try_to_proto_args(+ &self,+ end: &Self,+ step: &Self::StepType,+ include_end: bool,+ name: &str,+ ) -> Result<Option<datafusion_proto_models::protobuf::generate_series_node::Args>>+ {+ use datafusion_proto_models::protobuf;+ let step = Some(datafusion_proto_common::IntervalMonthDayNanoValue {+ months: step.months,+ days: step.days,+ nanos: step.nanoseconds,+ });+ let name = proto::encode_name(name)? as i32;+ let args = match &self.tz_str {+ Some(tz) => protobuf::generate_series_node::Args::TimestampArgs(+ protobuf::GenerateSeriesArgsTimestamp {+ start: self.value,+ end: end.value,+ step,+ include_end,+ name,+ tz: Some(tz.to_string()),+ },+ ),+ None => protobuf::generate_series_node::Args::DateArgs(+ protobuf::GenerateSeriesArgsDate {+ start: self.value,+ end: end.value,+ step,+ include_end,+ name,+ },+ ),+ };+ Ok(Some(args))+ }}/// Indicates the arguments used for generating a series.@@ -509,6 +606,33 @@ impl<T: SeriesValue> LazyBatchGenerator for GenericSeriesState<T> {new.finished = false;Arc::new(RwLock::new(new))}++ #[cfg(feature = "proto")]+ fn try_to_proto(+ &self,+ ) -> Result<Option<datafusion_proto_models::protobuf::GenerateSeriesNode>> {+ use datafusion_common::utils::usize_to_wire;+ use datafusion_proto_models::protobuf;++ let Some(args) = self.start.try_to_proto_args(+ &self.end,+ &self.step,+ self.include_end,+ self.name,+ )?+ else {+ return Ok(None);+ };+ Ok(Some(protobuf::GenerateSeriesNode {+ schema: None,+ target_batch_size: usize_to_wire(+ self.batch_size,+ "GenerateSeriesNode",+ "target_batch_size",+ )?,+ args: Some(args),+ }))+ }}impl<T: SeriesValue> fmt::Display for GenericSeriesState<T> {diff --git a/datafusion/functions-table/src/generate_series/proto.rs b/datafusion/functions-table/src/generate_series/proto.rsnew file mode 100644index 0000000000..5b2da6b586--- /dev/null+++ b/datafusion/functions-table/src/generate_series/proto.rs@@ -0,0 +1,120 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! The built-in generate-series wire format. Encoding is dispatched through+//! the generator hook; decoding stays with the types that own the payload.++use std::sync::Arc;++use arrow::datatypes::{IntervalMonthDayNanoType, Schema};+use datafusion_common::utils::usize_from_wire;+use datafusion_common::{Result, internal_datafusion_err, internal_err, plan_err};+use datafusion_physical_plan::ExecutionPlan;+use datafusion_physical_plan::memory::LazyMemoryExec;+use datafusion_proto_models::protobuf;++use super::{GenSeriesArgs, GenerateSeriesTable};++pub(super) fn encode_name(name: &str) -> Result<protobuf::GenerateSeriesName> {+ match name {+ "generate_series" => Ok(protobuf::GenerateSeriesName::GsGenerateSeries),+ "range" => Ok(protobuf::GenerateSeriesName::GsRange),+ _ => internal_err!("unknown name: {name}"),+ }+}++fn decode_name(name: protobuf::GenerateSeriesName) -> &'static str {+ match name {+ protobuf::GenerateSeriesName::GsGenerateSeries => "generate_series",+ protobuf::GenerateSeriesName::GsRange => "range",+ }+}++/// Reconstruct a single-generator [`LazyMemoryExec`] from its built-in payload.+///+/// This is the fixed dispatch entry point used by `datafusion-proto`; no+/// registry or extension codec is required for the built-in series types.+pub fn try_from_proto(+ node: &protobuf::GenerateSeriesNode,+) -> Result<Arc<dyn ExecutionPlan>> {+ let schema = node.schema.as_ref().ok_or_else(|| {+ internal_datafusion_err!("GenerateSeriesNode is missing required field 'schema'")+ })?;+ let schema = Arc::new(Schema::try_from(schema)?);+ let args = match &node.args {+ Some(protobuf::generate_series_node::Args::ContainsNull(args)) => {+ GenSeriesArgs::ContainsNull {+ name: decode_name(args.name()),+ }+ }+ Some(protobuf::generate_series_node::Args::Int64Args(args)) => {+ GenSeriesArgs::Int64Args {+ start: args.start,+ end: args.end,+ step: args.step,+ include_end: args.include_end,+ name: decode_name(args.name()),+ }+ }+ Some(protobuf::generate_series_node::Args::TimestampArgs(args)) => {+ let step = args.step.as_ref().ok_or_else(|| {+ internal_datafusion_err!("Missing step in TimestampArgs")+ })?;+ GenSeriesArgs::TimestampArgs {+ start: args.start,+ end: args.end,+ step: IntervalMonthDayNanoType::make_value(+ step.months,+ step.days,+ step.nanos,+ ),+ tz: args.tz.as_ref().map(|s| Arc::from(s.as_str())),+ include_end: args.include_end,+ name: decode_name(args.name()),+ }+ }+ Some(protobuf::generate_series_node::Args::DateArgs(args)) => {+ let step = args+ .step+ .as_ref()+ .ok_or_else(|| internal_datafusion_err!("Missing step in DateArgs"))?;+ GenSeriesArgs::DateArgs {+ start: args.start,+ end: args.end,+ step: IntervalMonthDayNanoType::make_value(+ step.months,+ step.days,+ step.nanos,+ ),+ include_end: args.include_end,+ name: decode_name(args.name()),+ }+ }+ None => return internal_err!("Missing args in GenerateSeriesNode"),+ };+ let target_batch_size = usize_from_wire(+ node.target_batch_size,+ "GenerateSeriesNode",+ "target_batch_size",+ )?;+ if target_batch_size == 0 {+ return plan_err!("GenerateSeriesNode: target_batch_size must be greater than 0");+ }+ let table = GenerateSeriesTable::new(Arc::clone(&schema), args);+ let generator = table.as_generator(target_batch_size)?;+ Ok(Arc::new(LazyMemoryExec::try_new(schema, vec![generator])?))+}diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rsindex 0b6bdf4490..64bedc8977 100644--- a/datafusion/physical-plan/src/memory.rs+++ b/datafusion/physical-plan/src/memory.rs@@ -159,6 +159,23 @@ pub trait LazyBatchGenerator: Send + Sync + fmt::Debug + fmt::Display {/// Returns a new instance with the state reset.fn reset_state(&self) -> Arc<RwLock<dyn LazyBatchGenerator>>;++ /// Encode this generator using the built-in generate-series wire format.+ ///+ /// The returned node carries the generator's arguments and batch size;+ /// [`LazyMemoryExec`] supplies its output schema. The current wire format+ /// only represents a single generate-series generator. Returning `Some(node)`+ /// opts into reconstruction as a built-in series generator: its rows, batch+ /// boundaries, and schema must match this generator after `reset_state()`.+ /// The payload must describe the initial state, not the current cursor.+ /// Returning `Ok(None)` leaves serialization to the execution plan's extension+ /// codec. Returning `Err` stops serialization without extension fallback.+ #[cfg(feature = "proto")]+ fn try_to_proto(+ &self,+ ) -> Result<Option<datafusion_proto_models::protobuf::GenerateSeriesNode>> {+ Ok(None)+ }}/// Execution plan for lazy in-memory batches of data@@ -383,6 +400,29 @@ impl ExecutionPlan for LazyMemoryExec {Some(self.metrics.clone_inner())}+ #[cfg(feature = "proto")]+ fn try_to_proto(+ &self,+ _ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,+ ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {+ use datafusion_proto_models::protobuf;++ // Preserve the existing single-generator wire format. Multiple+ // generators still require a plan extension codec.+ let [generator] = self.batch_generators.as_slice() else {+ return Ok(None);+ };+ let Some(mut node) = generator.read().try_to_proto()? else {+ return Ok(None);+ };+ node.schema = Some(self.schema.as_ref().try_into()?);+ Ok(Some(protobuf::PhysicalPlanNode {+ physical_plan_type: Some(+ protobuf::physical_plan_node::PhysicalPlanType::GenerateSeries(node),+ ),+ }))+ }+fn reset_state(self: Arc<Self>) -> Result<Arc<dyn ExecutionPlan>> {let generators = self.generators()diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.tomlindex 4b11bea011..2d678faba1 100644--- a/datafusion/proto/Cargo.toml+++ b/datafusion/proto/Cargo.toml@@ -68,7 +68,7 @@ datafusion-datasource-json = { workspace = true, features = ["proto"] }datafusion-datasource-parquet = { workspace = true, optional = true, features = ["proto"] }datafusion-execution = { workspace = true }datafusion-expr = { workspace = true, features = ["proto"] }-datafusion-functions-table = { workspace = true }+datafusion-functions-table = { workspace = true, features = ["proto"] }datafusion-physical-expr = { workspace = true, features = ["proto"] }datafusion-physical-expr-common = { workspace = true, features = ["proto"] }datafusion-physical-plan = { workspace = true, features = ["proto"] }@@ -93,6 +93,7 @@ datafusion-functions-aggregate = { workspace = true }datafusion-functions-window-common = { workspace = true }doc-comment = { workspace = true }flate2 = { workspace = true }+parking_lot = { workspace = true }pretty_assertions = "1.4"tempfile = { workspace = true }tokio = { workspace = true, features = ["rt-multi-thread"] }diff --git a/datafusion/proto/README.md b/datafusion/proto/README.mdindex c1382c5b8f..11d9101e71 100644--- a/datafusion/proto/README.md+++ b/datafusion/proto/README.md@@ -27,6 +27,26 @@ query engine.See [API Docs] for details and examples.+## Migrating generate-series serialization helpers++Generate-series encoding now uses `LazyBatchGenerator::try_to_proto`, with the+`proto` feature enabled. Existing `PhysicalPlanNodeExt` helpers remain available+as deprecated compatibility methods:++- Replace `try_into_generate_series_physical_plan` with+ `datafusion_functions_table::generate_series::proto::try_from_proto`.+- Replace `try_from_lazy_memory_exec` with `ExecutionPlan::try_to_proto` when+ handling `Ok(None)` yourself, or `AsExecutionPlan::try_from_physical_plan` for+ serialization with an extension codec.+- Replace `generate_series_name_to_str` and `str_to_generate_series_name` with+ explicit matches on `GenerateSeriesName::{GsGenerateSeries, GsRange}`.++A custom generator returning `Some(node)` is reconstructed as a built-in series+generator. Its rows, batch boundaries, and schema after reset must match that+built-in generator. Encode the initial state, not a partially consumed cursor.+`Ok(None)` permits extension fallback; `Err` stops serialization. The existing+single-generator protobuf wire format is unchanged.+[apache arrow]: https://arrow.apache.org/[apache datafusion]: https://datafusion.apache.org/[protocol buffers]: https://protobuf.dev/diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex e9b7fd647b..ecf5b70796 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -21,12 +21,10 @@ use std::collections::HashMap;use std::fmt::Debug;use std::sync::Arc;-use arrow::datatypes::{IntervalMonthDayNanoType, Schema, SchemaRef};+use arrow::datatypes::Schema;use datafusion_catalog::memory::MemorySourceConfig;-use datafusion_common::utils::{usize_from_wire, usize_to_wire};use datafusion_common::{DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err,- plan_err,};use datafusion_datasource_arrow::source::ArrowSource;#[cfg(feature = "avro")]@@ -42,9 +40,7 @@ use datafusion_datasource_parquet::source::ParquetSource;use datafusion_execution::{FunctionRegistry, TaskContext};use datafusion_expr::physical_planning_context::ScalarSubqueryResults;use datafusion_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF};-use datafusion_functions_table::generate_series::{- Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue,-};+use datafusion_functions_table::generate_series;use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;use datafusion_physical_plan::aggregates::AggregateExec;@@ -85,7 +81,6 @@ use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr};use prost::Message;use prost::bytes::BufMut;-use crate::convert_required;use crate::physical_plan::from_proto::parse_physical_expr_with_converter;use crate::physical_plan::to_proto::serialize_physical_expr_with_converter;use crate::protobuf::physical_plan_node::PhysicalPlanType;@@ -1153,6 +1148,48 @@ pub trait PhysicalPlanNodeExt: Sized {/// Returns a reference to the underlying [`protobuf::PhysicalPlanNode`].fn node(&self) -> &protobuf::PhysicalPlanNode;+ /// Compatibility helper for callers migrating to generator-owned encoding.+ #[deprecated(note = "Match GenerateSeriesName directly instead")]+ fn generate_series_name_to_str(name: protobuf::GenerateSeriesName) -> &'static str {+ match name {+ protobuf::GenerateSeriesName::GsGenerateSeries => "generate_series",+ protobuf::GenerateSeriesName::GsRange => "range",+ }+ }++ /// Decode the legacy payload through the generator-owned implementation.+ #[deprecated(note = "Use generate_series::proto::try_from_proto instead")]+ fn try_into_generate_series_physical_plan(+ &self,+ generate_series: &protobuf::GenerateSeriesNode,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ generate_series::proto::try_from_proto(generate_series)+ }++ /// Compatibility helper for callers migrating to generator-owned encoding.+ #[deprecated(note = "Use GenerateSeriesName variants directly instead")]+ fn str_to_generate_series_name(name: &str) -> Result<protobuf::GenerateSeriesName> {+ match name {+ "generate_series" => Ok(protobuf::GenerateSeriesName::GsGenerateSeries),+ "range" => Ok(protobuf::GenerateSeriesName::GsRange),+ _ => internal_err!("unknown name: {name}"),+ }+ }++ /// Encode through the plan hook, preserving unsupported-generator fallback.+ #[deprecated(+ note = "Use ExecutionPlan::try_to_proto or AsExecutionPlan::try_from_physical_plan instead"+ )]+ fn try_from_lazy_memory_exec(+ exec: &LazyMemoryExec,+ ) -> Result<Option<protobuf::PhysicalPlanNode>> {+ let encoder = ConverterPlanEncoder {+ codec: &DefaultPhysicalExtensionCodec {},+ proto_converter: &DefaultPhysicalProtoConverter {},+ };+ exec.try_to_proto(&ExecutionPlanEncodeCtx::new(&encoder))+ }+fn try_into_physical_plan_with_converter(&self,ctx: &TaskContext,@@ -1306,7 +1343,7 @@ pub trait PhysicalPlanNodeExt: Sized {CooperativeExec::try_from_proto(self.node(), &decode_ctx)}PhysicalPlanType::GenerateSeries(generate_series) => {- self.try_into_generate_series_physical_plan(generate_series)+ generate_series::proto::try_from_proto(generate_series)}PhysicalPlanType::SortMergeJoin(_) => {SortMergeJoinExec::try_from_proto(self.node(), &decode_ctx)@@ -1336,18 +1373,13 @@ pub trait PhysicalPlanNodeExt: Sized {) -> Result<protobuf::PhysicalPlanNode> {let plan_clone = Arc::clone(&plan);let mut plan = plan.as_ref();- // Resolve the downcast identity first so wrapper plans serialize as- // their delegate, matching how the `downcast_ref` chain below sees- // them. Without this a wrapper around a migrated plan would hit the- // wrapper's default `try_to_proto` (`Ok(None)`) and find no fallback- // arm for the delegate.+ // Resolve wrapper plans to their delegate before calling its hook.while let Some(delegate) = plan.downcast_delegate() {plan = delegate;}- // Self-serializing plans handle themselves via the `try_to_proto` hook- // (#22419). `Ok(None)` means "not migrated" and falls through to the- // central downcast chain below.+ // Built-in plans serialize through their `try_to_proto` hook.+ // `Ok(None)` falls through to the extension codec.let encoder = ConverterPlanEncoder {codec,proto_converter,@@ -1357,13 +1389,6 @@ pub trait PhysicalPlanNodeExt: Sized {return Ok(node);}- if let Some(exec) = plan.downcast_ref::<LazyMemoryExec>()- && let Some(node) =- protobuf::PhysicalPlanNode::try_from_lazy_memory_exec(exec)?- {- return Ok(node);- }-let mut buf: Vec<u8> = vec![];match codec.try_encode(Arc::clone(&plan_clone), &mut buf, proto_converter) {Ok(_) => {@@ -1456,221 +1481,6 @@ pub trait PhysicalPlanNodeExt: Sized {None => e,})}-- fn generate_series_name_to_str(name: protobuf::GenerateSeriesName) -> &'static str {- match name {- protobuf::GenerateSeriesName::GsGenerateSeries => "generate_series",- protobuf::GenerateSeriesName::GsRange => "range",- }- }-- fn try_into_generate_series_physical_plan(- &self,- generate_series: &protobuf::GenerateSeriesNode,- ) -> Result<Arc<dyn ExecutionPlan>> {- let schema: SchemaRef = Arc::new(convert_required!(generate_series.schema)?);-- let args = match &generate_series.args {- Some(protobuf::generate_series_node::Args::ContainsNull(args)) => {- GenSeriesArgs::ContainsNull {- name: protobuf::PhysicalPlanNode::generate_series_name_to_str(- args.name(),- ),- }- }- Some(protobuf::generate_series_node::Args::Int64Args(args)) => {- GenSeriesArgs::Int64Args {- start: args.start,- end: args.end,- step: args.step,- include_end: args.include_end,- name: protobuf::PhysicalPlanNode::generate_series_name_to_str(- args.name(),- ),- }- }- Some(protobuf::generate_series_node::Args::TimestampArgs(args)) => {- let step_proto = args.step.as_ref().ok_or_else(|| {- internal_datafusion_err!("Missing step in TimestampArgs")- })?;- let step = IntervalMonthDayNanoType::make_value(- step_proto.months,- step_proto.days,- step_proto.nanos,- );- GenSeriesArgs::TimestampArgs {- start: args.start,- end: args.end,- step,- tz: args.tz.as_ref().map(|s| Arc::from(s.as_str())),- include_end: args.include_end,- name: protobuf::PhysicalPlanNode::generate_series_name_to_str(- args.name(),- ),- }- }- Some(protobuf::generate_series_node::Args::DateArgs(args)) => {- let step_proto = args.step.as_ref().ok_or_else(|| {- internal_datafusion_err!("Missing step in DateArgs")- })?;- let step = IntervalMonthDayNanoType::make_value(- step_proto.months,- step_proto.days,- step_proto.nanos,- );- GenSeriesArgs::DateArgs {- start: args.start,- end: args.end,- step,- include_end: args.include_end,- name: protobuf::PhysicalPlanNode::generate_series_name_to_str(- args.name(),- ),- }- }- None => return internal_err!("Missing args in GenerateSeriesNode"),- };-- let table = GenerateSeriesTable::new(Arc::clone(&schema), args);- let target_batch_size = usize_from_wire(- generate_series.target_batch_size,- "GenerateSeriesNode",- "target_batch_size",- )?;- if target_batch_size == 0 {- return plan_err!(- "GenerateSeriesNode: target_batch_size must be greater than 0"- );- }- let generator = table.as_generator(target_batch_size)?;-- Ok(Arc::new(LazyMemoryExec::try_new(schema, vec![generator])?))- }-- fn str_to_generate_series_name(name: &str) -> Result<protobuf::GenerateSeriesName> {- match name {- "generate_series" => Ok(protobuf::GenerateSeriesName::GsGenerateSeries),- "range" => Ok(protobuf::GenerateSeriesName::GsRange),- _ => internal_err!("unknown name: {name}"),- }- }-- fn try_from_lazy_memory_exec(- exec: &LazyMemoryExec,- ) -> Result<Option<protobuf::PhysicalPlanNode>> {- let generators = exec.generators();-- // ensure we only have one generator- let [generator] = generators.as_slice() else {- return Ok(None);- };-- let generator_guard = generator.read();-- // Try to downcast to different generate_series types- if let Some(empty_gen) = generator_guard.as_any().downcast_ref::<Empty>() {- let schema = exec.schema();- let node = protobuf::GenerateSeriesNode {- schema: Some(schema.as_ref().try_into()?),- target_batch_size: 8192, // Default batch size- args: Some(protobuf::generate_series_node::Args::ContainsNull(- protobuf::GenerateSeriesArgsContainsNull {- name: protobuf::PhysicalPlanNode::str_to_generate_series_name(- empty_gen.name(),- )? as i32,- },- )),- };-- return Ok(Some(protobuf::PhysicalPlanNode {- physical_plan_type: Some(PhysicalPlanType::GenerateSeries(node)),- }));- }-- let encode_target_batch_size =- |size| usize_to_wire::<u32>(size, "GenerateSeriesNode", "target_batch_size");- if let Some(int_64) = generator_guard- .as_any()- .downcast_ref::<GenericSeriesState<i64>>()- {- let schema = exec.schema();- let node = protobuf::GenerateSeriesNode {- schema: Some(schema.as_ref().try_into()?),- target_batch_size: encode_target_batch_size(int_64.batch_size())?,- args: Some(protobuf::generate_series_node::Args::Int64Args(- protobuf::GenerateSeriesArgsInt64 {- start: *int_64.start(),- end: *int_64.end(),- step: *int_64.step(),- include_end: int_64.include_end(),- name: protobuf::PhysicalPlanNode::str_to_generate_series_name(- int_64.name(),- )? as i32,- },- )),- };-- return Ok(Some(protobuf::PhysicalPlanNode {- physical_plan_type: Some(PhysicalPlanType::GenerateSeries(node)),- }));- }-- if let Some(timestamp_args) = generator_guard- .as_any()- .downcast_ref::<GenericSeriesState<TimestampValue>>()- {- let schema = exec.schema();-- let start = timestamp_args.start().value();- let end = timestamp_args.end().value();-- let step_value = timestamp_args.step();-- let step = Some(datafusion_proto_common::IntervalMonthDayNanoValue {- months: step_value.months,- days: step_value.days,- nanos: step_value.nanoseconds,- });- let include_end = timestamp_args.include_end();- let name = protobuf::PhysicalPlanNode::str_to_generate_series_name(- timestamp_args.name(),- )? as i32;-- let args = match timestamp_args.current().tz_str() {- Some(tz) => protobuf::generate_series_node::Args::TimestampArgs(- protobuf::GenerateSeriesArgsTimestamp {- start,- end,- step,- include_end,- name,- tz: Some(tz.to_string()),- },- ),- None => protobuf::generate_series_node::Args::DateArgs(- protobuf::GenerateSeriesArgsDate {- start,- end,- step,- include_end,- name,- },- ),- };-- let node = protobuf::GenerateSeriesNode {- schema: Some(schema.as_ref().try_into()?),- target_batch_size: encode_target_batch_size(timestamp_args.batch_size())?,- args: Some(args),- };-- return Ok(Some(protobuf::PhysicalPlanNode {- physical_plan_type: Some(PhysicalPlanType::GenerateSeries(node)),- }));- }-- Ok(None)- }}impl PhysicalPlanNodeExt for protobuf::PhysicalPlanNode {diff --git a/datafusion/proto/tests/cases/plans/lazy_memory.rs b/datafusion/proto/tests/cases/plans/lazy_memory.rsnew file mode 100644index 0000000000..9d80aa03d0--- /dev/null+++ b/datafusion/proto/tests/cases/plans/lazy_memory.rs@@ -0,0 +1,436 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::any::Any;+use std::fmt;+use std::sync::Arc;++use arrow::array::{Int64Array, RecordBatch, TimestampNanosecondArray};+use arrow::datatypes::{DataType, Field, Schema, TimeUnit};+use datafusion_common::{Result, assert_contains, not_impl_err};+use datafusion_execution::TaskContext;+use datafusion_physical_plan::ExecutionPlan;+use datafusion_physical_plan::common::collect;+use datafusion_physical_plan::memory::{LazyBatchGenerator, LazyMemoryExec};+use datafusion_proto::physical_plan::{+ AsExecutionPlan, DefaultPhysicalExtensionCodec, PhysicalExtensionCodec,+ PhysicalPlanNodeExt, PhysicalProtoConverterExtension,+};+use datafusion_proto::protobuf::{self, PhysicalPlanNode};+use parking_lot::RwLock;+use prost::Message;++fn plan_node(node: protobuf::GenerateSeriesNode) -> PhysicalPlanNode {+ PhysicalPlanNode {+ physical_plan_type: Some(+ protobuf::physical_plan_node::PhysicalPlanType::GenerateSeries(node),+ ),+ }+}++fn int_node() -> Result<protobuf::GenerateSeriesNode> {+ let schema = Schema::new(vec![Field::new("v", DataType::Int64, false)]);+ Ok(protobuf::GenerateSeriesNode {+ schema: Some((&schema).try_into()?),+ target_batch_size: 2,+ args: Some(protobuf::generate_series_node::Args::Int64Args(+ protobuf::GenerateSeriesArgsInt64 {+ start: 3,+ end: 1,+ step: -1,+ include_end: true,+ name: protobuf::GenerateSeriesName::GsGenerateSeries as i32,+ },+ )),+ })+}++/// Construct the legacy wire messages independently of the generator hooks,+/// then check exact re-encoding and actual execution, including batch sizes.+#[tokio::test]+async fn lazy_memory_legacy_wire_roundtrip() -> Result<()> {+ use protobuf::generate_series_node::Args;+ for (name, include_end) in [+ (protobuf::GenerateSeriesName::GsGenerateSeries, true),+ (protobuf::GenerateSeriesName::GsRange, false),+ ] {+ let mut integer = int_node()?;+ let Some(Args::Int64Args(args)) = integer.args.as_mut() else {+ unreachable!()+ };+ args.name = name as i32;+ args.include_end = include_end;+ let mut empty = integer.clone();+ empty.target_batch_size = 8192;+ empty.args = Some(Args::ContainsNull(+ protobuf::GenerateSeriesArgsContainsNull { name: name as i32 },+ ));+ let expected = if include_end {+ vec![3, 2, 1]+ } else {+ vec![3, 2]+ };+ let mut cases = vec![(integer, expected.clone()), (empty, vec![])];+ for tz in [None, Some("+08:00")] {+ let schema = Schema::new(vec![Field::new(+ "v",+ DataType::Timestamp(TimeUnit::Nanosecond, tz.map(Into::into)),+ false,+ )]);+ let step = Some(datafusion_proto_common::IntervalMonthDayNanoValue {+ months: 0,+ days: 0,+ nanos: -1,+ });+ let args = match tz {+ Some(tz) => Args::TimestampArgs(protobuf::GenerateSeriesArgsTimestamp {+ start: 3,+ end: 1,+ step,+ include_end,+ name: name as i32,+ tz: Some(tz.to_string()),+ }),+ None => Args::DateArgs(protobuf::GenerateSeriesArgsDate {+ start: 3,+ end: 1,+ step,+ include_end,+ name: name as i32,+ }),+ };+ cases.push((+ protobuf::GenerateSeriesNode {+ schema: Some((&schema).try_into()?),+ target_batch_size: 2,+ args: Some(args),+ },+ expected.clone(),+ ));+ }+ for (node, expected) in cases {+ let legacy = plan_node(node);+ let bytes = legacy.encode_to_vec();+ let decoded = PhysicalPlanNode::decode(bytes.as_slice()).unwrap();+ let ctx = Arc::new(TaskContext::default());+ let plan = decoded+ .try_into_physical_plan(&ctx, &DefaultPhysicalExtensionCodec {})?;+ let encoded = PhysicalPlanNode::try_from_physical_plan(+ Arc::clone(&plan),+ &DefaultPhysicalExtensionCodec {},+ )?;+ assert_eq!(encoded.encode_to_vec(), bytes);+ // Repeated execute calls must still produce fresh independent streams.+ for _ in 0..2 {+ let batches = collect(plan.execute(0, Arc::clone(&ctx))?).await?;+ let sizes: Vec<_> = batches.iter().map(RecordBatch::num_rows).collect();+ let expected_sizes: Vec<_> =+ expected.chunks(2).map(<[i64]>::len).collect();+ assert_eq!(sizes, expected_sizes);+ let actual: Vec<i64> = batches+ .iter()+ .flat_map(|batch| {+ let column = batch.column(0);+ if let Some(array) = column.as_any().downcast_ref::<Int64Array>()+ {+ array.values().to_vec()+ } else {+ column+ .as_any()+ .downcast_ref::<TimestampNanosecondArray>()+ .unwrap()+ .values()+ .to_vec()+ }+ })+ .collect();+ assert_eq!(actual, expected);+ }+ }+ }+ Ok(())+}++#[derive(Debug)]+struct HookGenerator {+ payload: protobuf::GenerateSeriesNode,+ inner: Arc<RwLock<dyn LazyBatchGenerator>>,+}++impl HookGenerator {+ fn new(payload: protobuf::GenerateSeriesNode) -> Result<Self> {+ let mut executable = payload.clone();+ executable.schema = int_node()?.schema;+ let plan = datafusion_functions_table::generate_series::proto::try_from_proto(+ &executable,+ )?;+ let exec = plan.downcast_ref::<LazyMemoryExec>().unwrap();+ Ok(Self {+ payload,+ inner: Arc::clone(&exec.generators()[0]),+ })+ }+}++impl fmt::Display for HookGenerator {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ write!(f, "HookGenerator")+ }+}++impl LazyBatchGenerator for HookGenerator {+ fn as_any(&self) -> &dyn Any {+ self+ }+ fn generate_next_batch(&mut self) -> Result<Option<RecordBatch>> {+ self.inner.write().generate_next_batch()+ }+ fn reset_state(&self) -> Arc<RwLock<dyn LazyBatchGenerator>> {+ Arc::new(RwLock::new(Self {+ payload: self.payload.clone(),+ inner: self.inner.read().reset_state(),+ }))+ }+ fn try_to_proto(&self) -> Result<Option<protobuf::GenerateSeriesNode>> {+ Ok(Some(self.payload.clone()))+ }+}++#[tokio::test]+async fn lazy_memory_dispatches_to_generator_hook() -> Result<()> {+ let mut payload = int_node()?;+ // The plan, rather than the generator, supplies the authoritative schema.+ payload.schema = None;+ let generator = Arc::new(RwLock::new(HookGenerator::new(payload)?));+ // Consume the original cursor: serialization must still describe reset state.+ assert_eq!(+ generator.write().generate_next_batch()?.unwrap().num_rows(),+ 2+ );+ let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int64, false)]));+ let plan: Arc<dyn ExecutionPlan> =+ Arc::new(LazyMemoryExec::try_new(schema, vec![generator])?);+ let encoded = PhysicalPlanNode::try_from_physical_plan(+ Arc::clone(&plan),+ &DefaultPhysicalExtensionCodec {},+ )?;+ assert_eq!(encoded, plan_node(int_node()?));+ let ctx = Arc::new(TaskContext::default());+ let decoded =+ encoded.try_into_physical_plan(&ctx, &DefaultPhysicalExtensionCodec {})?;+ assert_eq!(decoded.schema(), plan.schema());+ for _ in 0..2 {+ let original = collect(plan.execute(0, Arc::clone(&ctx))?).await?;+ let roundtrip = collect(decoded.execute(0, Arc::clone(&ctx))?).await?;+ assert_eq!(original, roundtrip);+ assert_eq!(+ original+ .iter()+ .map(RecordBatch::num_rows)+ .collect::<Vec<_>>(),+ vec![2, 1]+ );+ let rows: Vec<i64> = original+ .iter()+ .flat_map(|batch| {+ batch+ .column(0)+ .as_any()+ .downcast_ref::<Int64Array>()+ .unwrap()+ .values()+ .to_vec()+ })+ .collect();+ assert_eq!(rows, vec![3, 2, 1]);+ }+ Ok(())+}++#[test]+#[expect(+ deprecated,+ reason = "Verify downstream callers of the compatibility methods"+)]+fn lazy_memory_legacy_helpers_forward_to_hooks() -> Result<()> {+ for (name, variant) in [+ (+ "generate_series",+ protobuf::GenerateSeriesName::GsGenerateSeries,+ ),+ ("range", protobuf::GenerateSeriesName::GsRange),+ ] {+ assert_eq!(PhysicalPlanNode::generate_series_name_to_str(variant), name);+ assert_eq!(+ PhysicalPlanNode::str_to_generate_series_name(name)?,+ variant+ );+ }+ assert!(PhysicalPlanNode::str_to_generate_series_name("unknown").is_err());+ let node = int_node()?;+ let plan = plan_node(node.clone()).try_into_generate_series_physical_plan(&node)?;+ let exec = plan.downcast_ref::<LazyMemoryExec>().unwrap();+ assert_eq!(+ PhysicalPlanNode::try_from_lazy_memory_exec(exec)?,+ Some(plan_node(node))+ );+ let opaque = LazyMemoryExec::try_new(+ Arc::new(Schema::empty()),+ vec![Arc::new(RwLock::new(OpaqueGenerator))],+ )?;+ assert_eq!(PhysicalPlanNode::try_from_lazy_memory_exec(&opaque)?, None);+ Ok(())+}++#[derive(Debug, Clone)]+struct OpaqueGenerator;++impl fmt::Display for OpaqueGenerator {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ write!(f, "OpaqueGenerator")+ }+}++// Deliberately use the default hook to cover existing downstream generators.+impl LazyBatchGenerator for OpaqueGenerator {+ fn as_any(&self) -> &dyn Any {+ self+ }+ fn generate_next_batch(&mut self) -> Result<Option<RecordBatch>> {+ Ok(None)+ }+ fn reset_state(&self) -> Arc<RwLock<dyn LazyBatchGenerator>> {+ Arc::new(RwLock::new(Self))+ }+}++#[derive(Debug)]+struct FallbackCodec;++impl PhysicalExtensionCodec for FallbackCodec {+ fn try_encode(+ &self,+ plan: Arc<dyn ExecutionPlan>,+ buf: &mut Vec<u8>,+ _converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<()> {+ assert!(plan.downcast_ref::<LazyMemoryExec>().is_some());+ buf.extend_from_slice(b"lazy-extension");+ Ok(())+ }+ fn try_decode(+ &self,+ _buf: &[u8],+ _inputs: &[Arc<dyn ExecutionPlan>],+ _ctx: &TaskContext,+ _converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ not_impl_err!("encoding-only test codec")+ }+}++#[test]+fn lazy_memory_preserves_extension_fallback() -> Result<()> {+ let opaque: Arc<RwLock<dyn LazyBatchGenerator>> =+ Arc::new(RwLock::new(OpaqueGenerator));+ let supported: Arc<RwLock<dyn LazyBatchGenerator>> =+ Arc::new(RwLock::new(HookGenerator::new(int_node()?)?));+ for generators in [+ vec![],+ vec![opaque],+ vec![Arc::clone(&supported), supported],+ ] {+ let schema = Arc::new(Schema::empty());+ let plan: Arc<dyn ExecutionPlan> =+ Arc::new(LazyMemoryExec::try_new(schema, generators)?);+ assert!(+ PhysicalPlanNode::try_from_physical_plan(+ Arc::clone(&plan),+ &DefaultPhysicalExtensionCodec {},+ )+ .is_err()+ );+ let node = PhysicalPlanNode::try_from_physical_plan(plan, &FallbackCodec)?;+ let Some(protobuf::physical_plan_node::PhysicalPlanType::Extension(extension)) =+ node.physical_plan_type+ else {+ panic!("expected extension fallback")+ };+ assert_eq!(extension.node, b"lazy-extension");+ }+ Ok(())+}++#[test]+#[cfg(target_pointer_width = "64")]+fn lazy_memory_preserves_encode_errors() -> Result<()> {+ use datafusion_functions_table::generate_series::{+ GenSeriesArgs, GenerateSeriesTable,+ };++ let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int64, false)]));+ let table = GenerateSeriesTable::new(+ Arc::clone(&schema),+ GenSeriesArgs::Int64Args {+ start: 1,+ end: 3,+ step: 1,+ include_end: true,+ name: "generate_series",+ },+ );+ let generator = table.as_generator(usize::MAX)?;+ let plan = Arc::new(LazyMemoryExec::try_new(schema, vec![generator])?);+ // A real encode error must propagate even if an extension codec is available.+ let error =+ PhysicalPlanNode::try_from_physical_plan(plan, &FallbackCodec).unwrap_err();+ assert_contains!(error.to_string(), "target_batch_size");+ assert_contains!(error.to_string(), "out of range for the plan wire format");+ Ok(())+}++#[test]+fn lazy_memory_rejects_malformed_series_payloads() -> Result<()> {+ let mut missing_schema = int_node()?;+ missing_schema.schema = None;+ let mut missing_args = int_node()?;+ missing_args.args = None;+ let mut zero_batch_size = int_node()?;+ zero_batch_size.target_batch_size = 0;+ let mut missing_step = int_node()?;+ missing_step.args = Some(protobuf::generate_series_node::Args::TimestampArgs(+ protobuf::GenerateSeriesArgsTimestamp {+ step: None,+ ..Default::default()+ },+ ));+ for (node, expected) in [+ (missing_schema, "schema"),+ (missing_args, "Missing args"),+ (zero_batch_size, "target_batch_size must be greater than 0"),+ (missing_step, "Missing step in TimestampArgs"),+ ] {+ let error = plan_node(node)+ .try_into_physical_plan(+ &TaskContext::default(),+ &DefaultPhysicalExtensionCodec {},+ )+ .unwrap_err();+ assert_contains!(error.to_string(), expected);+ }+ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/mod.rs b/datafusion/proto/tests/cases/plans/mod.rsindex 1faa634380..0f00f35581 100644--- a/datafusion/proto/tests/cases/plans/mod.rs+++ b/datafusion/proto/tests/cases/plans/mod.rs@@ -40,6 +40,7 @@ mod exprs;mod extensions;mod filters;mod joins;+mod lazy_memory;mod leaves;mod limits;mod misc;diff --git a/dev/changelog/14.0.0.md b/dev/changelog/14.0.0.mdindex 8020dd1924..78921a4bd0 100644--- a/dev/changelog/14.0.0.md+++ b/dev/changelog/14.0.0.md@@ -157,7 +157,7 @@- Exception: Internal error, Exception: Schema error [\#2938](https://github.com/apache/datafusion/issues/2938)- Version 0.6.0 Panic error during SQL execution [\#2738](https://github.com/apache/datafusion/issues/2738)- wrong result when operation parquet [\#2044](https://github.com/apache/datafusion/issues/2044)-- Local object store accepts file:/// as base path, but LocalStore returns meta without the prefix. [\#1923](https://github.com/apache/datafusion/issues/1923)+- Local object store accepts `file:///` as base path, but LocalStore returns meta without the prefix. [\#1923](https://github.com/apache/datafusion/issues/1923)- Reading nested parquet files results in `index out of bounds` [\#1383](https://github.com/apache/datafusion/issues/1383)- `-` \(negation\) with NULL literals does not work: can't be evaluated because the expression's type is Utf8, not signed [\#1192](https://github.com/apache/datafusion/issues/1192)- Inconsistent cast behavior [\#957](https://github.com/apache/datafusion/issues/957)