Apache DataFusion SQL Query Engine
Add protobuf serialization/deserialization support for `EmptyTable` scans (#20844)
Olehf802ed1c51ad
diff --git a/datafusion/core/src/datasource/empty.rs b/datafusion/catalog/src/empty.rssimilarity index 92%rename from datafusion/core/src/datasource/empty.rsrename to datafusion/catalog/src/empty.rsindex ef7f29d205..1ff36ecf36 100644--- a/datafusion/core/src/datasource/empty.rs+++ b/datafusion/catalog/src/empty.rs@@ -21,15 +21,14 @@ use std::sync::Arc;use arrow::datatypes::*;use async_trait::async_trait;-use datafusion_catalog::Session;-use datafusion_common::project_schema;--use crate::datasource::{TableProvider, TableType};-use crate::error::Result;-use crate::logical_expr::Expr;+use datafusion_common::{Result, project_schema};+use datafusion_expr::{Expr, TableType};use datafusion_physical_plan::ExecutionPlan;use datafusion_physical_plan::empty::EmptyExec;+use crate::Session;+use crate::TableProvider;+/// An empty plan that is useful for testing and generating plans/// without mapping them to actual data.#[derive(Debug)]diff --git a/datafusion/catalog/src/lib.rs b/datafusion/catalog/src/lib.rsindex 931941e8fd..33d54b7cb8 100644--- a/datafusion/catalog/src/lib.rs+++ b/datafusion/catalog/src/lib.rs@@ -34,6 +34,7 @@pub mod cte_worktable;pub mod default_table_source;+pub mod empty;pub mod information_schema;pub mod listing_schema;pub mod memory;diff --git a/datafusion/core/src/datasource/mod.rs b/datafusion/core/src/datasource/mod.rsindex 3fad65ae0d..de54078aaf 100644--- a/datafusion/core/src/datasource/mod.rs+++ b/datafusion/core/src/datasource/mod.rs@@ -20,7 +20,6 @@//! [`ListingTable`]: crate::datasource::listing::ListingTablepub mod dynamic_file;-pub mod empty;pub mod file_format;pub mod listing;pub mod listing_table_factory;@@ -39,6 +38,7 @@ pub use crate::catalog::TableProvider;pub use crate::logical_expr::TableType;pub use datafusion_catalog::cte_worktable;pub use datafusion_catalog::default_table_source;+pub use datafusion_catalog::empty;pub use datafusion_catalog::memory;pub use datafusion_catalog::stream;pub use datafusion_catalog::view;diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.protoindex 29b71929f1..5cd57b9f98 100644--- a/datafusion/proto/proto/datafusion.proto+++ b/datafusion/proto/proto/datafusion.proto@@ -62,6 +62,7 @@ message LogicalPlanNode {RecursiveQueryNode recursive_query = 31;CteWorkTableScanNode cte_work_table_scan = 32;DmlNode dml = 33;+ EmptyTableScanNode empty_table_scan = 34;}}@@ -1426,6 +1427,13 @@ message CteWorkTableScanNode {datafusion_common.Schema schema = 2;}+message EmptyTableScanNode {+ TableReference table_name = 1;+ datafusion_common.Schema schema = 2;+ ProjectionColumns projection = 3;+ repeated LogicalExprNode filters = 4;+}+enum GenerateSeriesName {GS_GENERATE_SERIES = 0;GS_RANGE = 1;diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rsindex d84fdae345..74f7b3b1c3 100644--- a/datafusion/proto/src/generated/pbjson.rs+++ b/datafusion/proto/src/generated/pbjson.rs@@ -5914,6 +5914,149 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {deserializer.deserialize_struct("datafusion.EmptyRelationNode", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for EmptyTableScanNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if self.table_name.is_some() {+ len += 1;+ }+ if self.schema.is_some() {+ len += 1;+ }+ if self.projection.is_some() {+ len += 1;+ }+ if !self.filters.is_empty() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.EmptyTableScanNode", len)?;+ if let Some(v) = self.table_name.as_ref() {+ struct_ser.serialize_field("tableName", v)?;+ }+ if let Some(v) = self.schema.as_ref() {+ struct_ser.serialize_field("schema", v)?;+ }+ if let Some(v) = self.projection.as_ref() {+ struct_ser.serialize_field("projection", v)?;+ }+ if !self.filters.is_empty() {+ struct_ser.serialize_field("filters", &self.filters)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for EmptyTableScanNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "table_name",+ "tableName",+ "schema",+ "projection",+ "filters",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ TableName,+ Schema,+ Projection,+ Filters,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "tableName" | "table_name" => Ok(GeneratedField::TableName),+ "schema" => Ok(GeneratedField::Schema),+ "projection" => Ok(GeneratedField::Projection),+ "filters" => Ok(GeneratedField::Filters),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = EmptyTableScanNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.EmptyTableScanNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<EmptyTableScanNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut table_name__ = None;+ let mut schema__ = None;+ let mut projection__ = None;+ let mut filters__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::TableName => {+ if table_name__.is_some() {+ return Err(serde::de::Error::duplicate_field("tableName"));+ }+ table_name__ = map_.next_value()?;+ }+ GeneratedField::Schema => {+ if schema__.is_some() {+ return Err(serde::de::Error::duplicate_field("schema"));+ }+ schema__ = map_.next_value()?;+ }+ GeneratedField::Projection => {+ if projection__.is_some() {+ return Err(serde::de::Error::duplicate_field("projection"));+ }+ projection__ = map_.next_value()?;+ }+ GeneratedField::Filters => {+ if filters__.is_some() {+ return Err(serde::de::Error::duplicate_field("filters"));+ }+ filters__ = Some(map_.next_value()?);+ }+ }+ }+ Ok(EmptyTableScanNode {+ table_name: table_name__,+ schema: schema__,+ projection: projection__,+ filters: filters__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.EmptyTableScanNode", FIELDS, GeneratedVisitor)+ }+}impl serde::Serialize for ExplainExecNode {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>@@ -12918,6 +13061,9 @@ impl serde::Serialize for LogicalPlanNode {logical_plan_node::LogicalPlanType::Dml(v) => {struct_ser.serialize_field("dml", v)?;}+ logical_plan_node::LogicalPlanType::EmptyTableScan(v) => {+ struct_ser.serialize_field("emptyTableScan", v)?;+ }}}struct_ser.end()@@ -12977,6 +13123,8 @@ impl<'de> serde::Deserialize<'de> for LogicalPlanNode {"cte_work_table_scan","cteWorkTableScan","dml",+ "empty_table_scan",+ "emptyTableScan",];#[allow(clippy::enum_variant_names)]@@ -13013,6 +13161,7 @@ impl<'de> serde::Deserialize<'de> for LogicalPlanNode {RecursiveQuery,CteWorkTableScan,Dml,+ EmptyTableScan,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -13066,6 +13215,7 @@ impl<'de> serde::Deserialize<'de> for LogicalPlanNode {"recursiveQuery" | "recursive_query" => Ok(GeneratedField::RecursiveQuery),"cteWorkTableScan" | "cte_work_table_scan" => Ok(GeneratedField::CteWorkTableScan),"dml" => Ok(GeneratedField::Dml),+ "emptyTableScan" | "empty_table_scan" => Ok(GeneratedField::EmptyTableScan),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -13310,6 +13460,13 @@ impl<'de> serde::Deserialize<'de> for LogicalPlanNode {return Err(serde::de::Error::duplicate_field("dml"));}logical_plan_type__ = map_.next_value::<::std::option::Option<_>>()?.map(logical_plan_node::LogicalPlanType::Dml)+;+ }+ GeneratedField::EmptyTableScan => {+ if logical_plan_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("emptyTableScan"));+ }+ logical_plan_type__ = map_.next_value::<::std::option::Option<_>>()?.map(logical_plan_node::LogicalPlanType::EmptyTableScan);}}diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rsindex 1dc259926a..3db2f2de8e 100644--- a/datafusion/proto/src/generated/prost.rs+++ b/datafusion/proto/src/generated/prost.rs@@ -5,7 +5,7 @@pub struct LogicalPlanNode {#[prost(oneof = "logical_plan_node::LogicalPlanType",- tags = "1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33"+ tags = "1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34")]pub logical_plan_type: ::core::option::Option<logical_plan_node::LogicalPlanType>,}@@ -77,6 +77,8 @@ pub mod logical_plan_node {CteWorkTableScan(super::CteWorkTableScanNode),#[prost(message, tag = "33")]Dml(::prost::alloc::boxed::Box<super::DmlNode>),+ #[prost(message, tag = "34")]+ EmptyTableScan(super::EmptyTableScanNode),}}#[derive(Clone, PartialEq, ::prost::Message)]@@ -2115,6 +2117,17 @@ pub struct CteWorkTableScanNode {#[prost(message, optional, tag = "2")]pub schema: ::core::option::Option<super::datafusion_common::Schema>,}+#[derive(Clone, PartialEq, ::prost::Message)]+pub struct EmptyTableScanNode {+ #[prost(message, optional, tag = "1")]+ pub table_name: ::core::option::Option<TableReference>,+ #[prost(message, optional, tag = "2")]+ pub schema: ::core::option::Option<super::datafusion_common::Schema>,+ #[prost(message, optional, tag = "3")]+ pub projection: ::core::option::Option<ProjectionColumns>,+ #[prost(message, repeated, tag = "4")]+ pub filters: ::prost::alloc::vec::Vec<LogicalExprNode>,+}#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]pub struct GenerateSeriesArgsContainsNull {#[prost(enumeration = "GenerateSeriesName", tag = "1")]diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex eea2eb3364..1417ba850b 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -35,6 +35,7 @@ use crate::{use crate::protobuf::{ToProtoError, proto_error};use arrow::datatypes::{DataType, Field, Schema, SchemaBuilder, SchemaRef};use datafusion_catalog::cte_worktable::CteWorkTable;+use datafusion_catalog::empty::EmptyTable;use datafusion_common::file_options::file_type::FileType;use datafusion_common::{Result, TableReference, ToDFSchema, assert_or_internal_err, context,@@ -1065,6 +1066,35 @@ impl AsLogicalPlan for LogicalPlanNode {)?.build()}+ LogicalPlanType::EmptyTableScan(scan) => {+ let schema: Schema = convert_required!(scan.schema)?;+ let schema = Arc::new(schema);+ let mut projection = None;+ if let Some(columns) = &scan.projection {+ let column_indices = columns+ .columns+ .iter()+ .map(|name| schema.index_of(name))+ .collect::<Result<Vec<usize>, _>>()?;+ projection = Some(column_indices);+ }++ let filters =+ from_proto::parse_exprs(&scan.filters, ctx, extension_codec)?;++ let table_name =+ from_table_reference(scan.table_name.as_ref(), "EmptyTableScan")?;++ let provider = Arc::new(EmptyTable::new(Arc::clone(&schema)));++ LogicalPlanBuilder::scan_with_filters(+ table_name,+ provider_as_source(provider),+ projection,+ filters,+ )?+ .build()+ }LogicalPlanType::Dml(dml_node) => {Ok(LogicalPlan::Dml(datafusion_expr::DmlStatement::new(from_table_reference(dml_node.table_name.as_ref(), "DML ")?,@@ -1277,6 +1307,19 @@ impl AsLogicalPlan for LogicalPlanNode {},)),})+ } else if provider.downcast_ref::<EmptyTable>().is_some() {+ let schema: protobuf::Schema = schema.as_ref().try_into()?;++ Ok(LogicalPlanNode {+ logical_plan_type: Some(LogicalPlanType::EmptyTableScan(+ protobuf::EmptyTableScanNode {+ table_name: Some(table_name.clone().into()),+ schema: Some(schema),+ projection,+ filters,+ },+ )),+ })} else {let schema: protobuf::Schema = schema.as_ref().try_into()?;let mut bytes = vec![];diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex c0570881e3..dbc95536f0 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -3099,3 +3099,55 @@ async fn roundtrip_mixed_case_table_reference() -> Result<()> {Ok(())}++#[tokio::test]+async fn roundtrip_empty_table_scan() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int32, false),+ Field::new("name", DataType::Utf8, true),+ ]));+ let table = Arc::new(datafusion::datasource::empty::EmptyTable::new(Arc::clone(+ &schema,+ )));++ let ctx = SessionContext::new();+ ctx.register_table("empty", table)?;++ let plan = ctx.table("empty").await?.into_optimized_plan()?;+ let bytes = logical_plan_to_bytes(&plan)?;+ let restored = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;++ assert_eq!(+ format!("{}", plan.display_indent_schema()),+ format!("{}", restored.display_indent_schema()),+ );+ Ok(())+}++#[tokio::test]+async fn roundtrip_empty_table_scan_with_projection() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int32, false),+ Field::new("name", DataType::Utf8, true),+ ]));+ let table = Arc::new(datafusion::datasource::empty::EmptyTable::new(Arc::clone(+ &schema,+ )));++ let ctx = SessionContext::new();+ ctx.register_table("empty", table)?;++ let plan = ctx+ .table("empty")+ .await?+ .select_columns(&["name"])?+ .into_optimized_plan()?;+ let bytes = logical_plan_to_bytes(&plan)?;+ let restored = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;++ assert_eq!(+ format!("{}", plan.display_indent_schema()),+ format!("{}", restored.display_indent_schema()),+ );+ Ok(())+}