Skip to content

Apache DataFusion SQL Query Engine

Add protobuf serialization/deserialization support for `EmptyTable` scans (#20844)

Olehf802ed1c51ad
diff --git a/datafusion/core/src/datasource/empty.rs b/datafusion/catalog/src/empty.rs
similarity index 92%
rename from datafusion/core/src/datasource/empty.rs
rename to datafusion/catalog/src/empty.rs
index ef7f29d205..1ff36ecf36 100644
--- a/datafusion/core/src/datasource/empty.rs
+++ b/datafusion/catalog/src/empty.rs
@@ -21,15 +21,14 @@ use std::sync::Arc;
use arrow::datatypes::*;
use async_trait::async_trait;
-use datafusion_catalog::Session;
-use datafusion_common::project_schema;
-
-use crate::datasource::{TableProvider, TableType};
-use crate::error::Result;
-use crate::logical_expr::Expr;
+use datafusion_common::{Result, project_schema};
+use datafusion_expr::{Expr, TableType};
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::empty::EmptyExec;
+use crate::Session;
+use crate::TableProvider;
+
/// An empty plan that is useful for testing and generating plans
/// without mapping them to actual data.
#[derive(Debug)]
diff --git a/datafusion/catalog/src/lib.rs b/datafusion/catalog/src/lib.rs
index 931941e8fd..33d54b7cb8 100644
--- a/datafusion/catalog/src/lib.rs
+++ b/datafusion/catalog/src/lib.rs
@@ -34,6 +34,7 @@
pub mod cte_worktable;
pub mod default_table_source;
+pub mod empty;
pub mod information_schema;
pub mod listing_schema;
pub mod memory;
diff --git a/datafusion/core/src/datasource/mod.rs b/datafusion/core/src/datasource/mod.rs
index 3fad65ae0d..de54078aaf 100644
--- a/datafusion/core/src/datasource/mod.rs
+++ b/datafusion/core/src/datasource/mod.rs
@@ -20,7 +20,6 @@
//! [`ListingTable`]: crate::datasource::listing::ListingTable
pub mod dynamic_file;
-pub mod empty;
pub mod file_format;
pub mod listing;
pub mod listing_table_factory;
@@ -39,6 +38,7 @@ pub use crate::catalog::TableProvider;
pub use crate::logical_expr::TableType;
pub use datafusion_catalog::cte_worktable;
pub use datafusion_catalog::default_table_source;
+pub use datafusion_catalog::empty;
pub use datafusion_catalog::memory;
pub use datafusion_catalog::stream;
pub use datafusion_catalog::view;
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto
index 29b71929f1..5cd57b9f98 100644
--- a/datafusion/proto/proto/datafusion.proto
+++ b/datafusion/proto/proto/datafusion.proto
@@ -62,6 +62,7 @@ message LogicalPlanNode {
RecursiveQueryNode recursive_query = 31;
CteWorkTableScanNode cte_work_table_scan = 32;
DmlNode dml = 33;
+ EmptyTableScanNode empty_table_scan = 34;
}
}
@@ -1426,6 +1427,13 @@ message CteWorkTableScanNode {
datafusion_common.Schema schema = 2;
}
+message EmptyTableScanNode {
+ TableReference table_name = 1;
+ datafusion_common.Schema schema = 2;
+ ProjectionColumns projection = 3;
+ repeated LogicalExprNode filters = 4;
+}
+
enum GenerateSeriesName {
GS_GENERATE_SERIES = 0;
GS_RANGE = 1;
diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs
index d84fdae345..74f7b3b1c3 100644
--- a/datafusion/proto/src/generated/pbjson.rs
+++ b/datafusion/proto/src/generated/pbjson.rs
@@ -5914,6 +5914,149 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {
deserializer.deserialize_struct("datafusion.EmptyRelationNode", FIELDS, GeneratedVisitor)
}
}
+impl serde::Serialize for EmptyTableScanNode {
+ #[allow(deprecated)]
+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ use serde::ser::SerializeStruct;
+ let mut len = 0;
+ if self.table_name.is_some() {
+ len += 1;
+ }
+ if self.schema.is_some() {
+ len += 1;
+ }
+ if self.projection.is_some() {
+ len += 1;
+ }
+ if !self.filters.is_empty() {
+ len += 1;
+ }
+ let mut struct_ser = serializer.serialize_struct("datafusion.EmptyTableScanNode", len)?;
+ if let Some(v) = self.table_name.as_ref() {
+ struct_ser.serialize_field("tableName", v)?;
+ }
+ if let Some(v) = self.schema.as_ref() {
+ struct_ser.serialize_field("schema", v)?;
+ }
+ if let Some(v) = self.projection.as_ref() {
+ struct_ser.serialize_field("projection", v)?;
+ }
+ if !self.filters.is_empty() {
+ struct_ser.serialize_field("filters", &self.filters)?;
+ }
+ struct_ser.end()
+ }
+}
+impl<'de> serde::Deserialize<'de> for EmptyTableScanNode {
+ #[allow(deprecated)]
+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ const FIELDS: &[&str] = &[
+ "table_name",
+ "tableName",
+ "schema",
+ "projection",
+ "filters",
+ ];
+
+ #[allow(clippy::enum_variant_names)]
+ enum GeneratedField {
+ TableName,
+ Schema,
+ Projection,
+ Filters,
+ }
+ impl<'de> serde::Deserialize<'de> for GeneratedField {
+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ struct GeneratedVisitor;
+
+ impl serde::de::Visitor<'_> for GeneratedVisitor {
+ type Value = GeneratedField;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(formatter, "expected one of: {:?}", &FIELDS)
+ }
+
+ #[allow(unused_variables)]
+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>
+ where
+ E: serde::de::Error,
+ {
+ match value {
+ "tableName" | "table_name" => Ok(GeneratedField::TableName),
+ "schema" => Ok(GeneratedField::Schema),
+ "projection" => Ok(GeneratedField::Projection),
+ "filters" => Ok(GeneratedField::Filters),
+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),
+ }
+ }
+ }
+ deserializer.deserialize_identifier(GeneratedVisitor)
+ }
+ }
+ struct GeneratedVisitor;
+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {
+ type Value = EmptyTableScanNode;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ formatter.write_str("struct datafusion.EmptyTableScanNode")
+ }
+
+ fn visit_map<V>(self, mut map_: V) -> std::result::Result<EmptyTableScanNode, V::Error>
+ where
+ V: serde::de::MapAccess<'de>,
+ {
+ let mut table_name__ = None;
+ let mut schema__ = None;
+ let mut projection__ = None;
+ let mut filters__ = None;
+ while let Some(k) = map_.next_key()? {
+ match k {
+ GeneratedField::TableName => {
+ if table_name__.is_some() {
+ return Err(serde::de::Error::duplicate_field("tableName"));
+ }
+ table_name__ = map_.next_value()?;
+ }
+ GeneratedField::Schema => {
+ if schema__.is_some() {
+ return Err(serde::de::Error::duplicate_field("schema"));
+ }
+ schema__ = map_.next_value()?;
+ }
+ GeneratedField::Projection => {
+ if projection__.is_some() {
+ return Err(serde::de::Error::duplicate_field("projection"));
+ }
+ projection__ = map_.next_value()?;
+ }
+ GeneratedField::Filters => {
+ if filters__.is_some() {
+ return Err(serde::de::Error::duplicate_field("filters"));
+ }
+ filters__ = Some(map_.next_value()?);
+ }
+ }
+ }
+ Ok(EmptyTableScanNode {
+ table_name: table_name__,
+ schema: schema__,
+ projection: projection__,
+ filters: filters__.unwrap_or_default(),
+ })
+ }
+ }
+ deserializer.deserialize_struct("datafusion.EmptyTableScanNode", FIELDS, GeneratedVisitor)
+ }
+}
impl serde::Serialize for ExplainExecNode {
#[allow(deprecated)]
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
@@ -12918,6 +13061,9 @@ impl serde::Serialize for LogicalPlanNode {
logical_plan_node::LogicalPlanType::Dml(v) => {
struct_ser.serialize_field("dml", v)?;
}
+ logical_plan_node::LogicalPlanType::EmptyTableScan(v) => {
+ struct_ser.serialize_field("emptyTableScan", v)?;
+ }
}
}
struct_ser.end()
@@ -12977,6 +13123,8 @@ impl<'de> serde::Deserialize<'de> for LogicalPlanNode {
"cte_work_table_scan",
"cteWorkTableScan",
"dml",
+ "empty_table_scan",
+ "emptyTableScan",
];
#[allow(clippy::enum_variant_names)]
@@ -13013,6 +13161,7 @@ impl<'de> serde::Deserialize<'de> for LogicalPlanNode {
RecursiveQuery,
CteWorkTableScan,
Dml,
+ EmptyTableScan,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -13066,6 +13215,7 @@ impl<'de> serde::Deserialize<'de> for LogicalPlanNode {
"recursiveQuery" | "recursive_query" => Ok(GeneratedField::RecursiveQuery),
"cteWorkTableScan" | "cte_work_table_scan" => Ok(GeneratedField::CteWorkTableScan),
"dml" => Ok(GeneratedField::Dml),
+ "emptyTableScan" | "empty_table_scan" => Ok(GeneratedField::EmptyTableScan),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -13310,6 +13460,13 @@ impl<'de> serde::Deserialize<'de> for LogicalPlanNode {
return Err(serde::de::Error::duplicate_field("dml"));
}
logical_plan_type__ = map_.next_value::<::std::option::Option<_>>()?.map(logical_plan_node::LogicalPlanType::Dml)
+;
+ }
+ GeneratedField::EmptyTableScan => {
+ if logical_plan_type__.is_some() {
+ return Err(serde::de::Error::duplicate_field("emptyTableScan"));
+ }
+ logical_plan_type__ = map_.next_value::<::std::option::Option<_>>()?.map(logical_plan_node::LogicalPlanType::EmptyTableScan)
;
}
}
diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs
index 1dc259926a..3db2f2de8e 100644
--- a/datafusion/proto/src/generated/prost.rs
+++ b/datafusion/proto/src/generated/prost.rs
@@ -5,7 +5,7 @@
pub struct LogicalPlanNode {
#[prost(
oneof = "logical_plan_node::LogicalPlanType",
- tags = "1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33"
+ tags = "1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34"
)]
pub logical_plan_type: ::core::option::Option<logical_plan_node::LogicalPlanType>,
}
@@ -77,6 +77,8 @@ pub mod logical_plan_node {
CteWorkTableScan(super::CteWorkTableScanNode),
#[prost(message, tag = "33")]
Dml(::prost::alloc::boxed::Box<super::DmlNode>),
+ #[prost(message, tag = "34")]
+ EmptyTableScan(super::EmptyTableScanNode),
}
}
#[derive(Clone, PartialEq, ::prost::Message)]
@@ -2115,6 +2117,17 @@ pub struct CteWorkTableScanNode {
#[prost(message, optional, tag = "2")]
pub schema: ::core::option::Option<super::datafusion_common::Schema>,
}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct EmptyTableScanNode {
+ #[prost(message, optional, tag = "1")]
+ pub table_name: ::core::option::Option<TableReference>,
+ #[prost(message, optional, tag = "2")]
+ pub schema: ::core::option::Option<super::datafusion_common::Schema>,
+ #[prost(message, optional, tag = "3")]
+ pub projection: ::core::option::Option<ProjectionColumns>,
+ #[prost(message, repeated, tag = "4")]
+ pub filters: ::prost::alloc::vec::Vec<LogicalExprNode>,
+}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GenerateSeriesArgsContainsNull {
#[prost(enumeration = "GenerateSeriesName", tag = "1")]
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
index eea2eb3364..1417ba850b 100644
--- a/datafusion/proto/src/logical_plan/mod.rs
+++ b/datafusion/proto/src/logical_plan/mod.rs
@@ -35,6 +35,7 @@ use crate::{
use crate::protobuf::{ToProtoError, proto_error};
use arrow::datatypes::{DataType, Field, Schema, SchemaBuilder, SchemaRef};
use datafusion_catalog::cte_worktable::CteWorkTable;
+use datafusion_catalog::empty::EmptyTable;
use datafusion_common::file_options::file_type::FileType;
use datafusion_common::{
Result, TableReference, ToDFSchema, assert_or_internal_err, context,
@@ -1065,6 +1066,35 @@ impl AsLogicalPlan for LogicalPlanNode {
)?
.build()
}
+ LogicalPlanType::EmptyTableScan(scan) => {
+ let schema: Schema = convert_required!(scan.schema)?;
+ let schema = Arc::new(schema);
+ let mut projection = None;
+ if let Some(columns) = &scan.projection {
+ let column_indices = columns
+ .columns
+ .iter()
+ .map(|name| schema.index_of(name))
+ .collect::<Result<Vec<usize>, _>>()?;
+ projection = Some(column_indices);
+ }
+
+ let filters =
+ from_proto::parse_exprs(&scan.filters, ctx, extension_codec)?;
+
+ let table_name =
+ from_table_reference(scan.table_name.as_ref(), "EmptyTableScan")?;
+
+ let provider = Arc::new(EmptyTable::new(Arc::clone(&schema)));
+
+ LogicalPlanBuilder::scan_with_filters(
+ table_name,
+ provider_as_source(provider),
+ projection,
+ filters,
+ )?
+ .build()
+ }
LogicalPlanType::Dml(dml_node) => {
Ok(LogicalPlan::Dml(datafusion_expr::DmlStatement::new(
from_table_reference(dml_node.table_name.as_ref(), "DML ")?,
@@ -1277,6 +1307,19 @@ impl AsLogicalPlan for LogicalPlanNode {
},
)),
})
+ } else if provider.downcast_ref::<EmptyTable>().is_some() {
+ let schema: protobuf::Schema = schema.as_ref().try_into()?;
+
+ Ok(LogicalPlanNode {
+ logical_plan_type: Some(LogicalPlanType::EmptyTableScan(
+ protobuf::EmptyTableScanNode {
+ table_name: Some(table_name.clone().into()),
+ schema: Some(schema),
+ projection,
+ filters,
+ },
+ )),
+ })
} else {
let schema: protobuf::Schema = schema.as_ref().try_into()?;
let mut bytes = vec![];
diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
index c0570881e3..dbc95536f0 100644
--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
@@ -3099,3 +3099,55 @@ async fn roundtrip_mixed_case_table_reference() -> Result<()> {
Ok(())
}
+
+#[tokio::test]
+async fn roundtrip_empty_table_scan() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new("name", DataType::Utf8, true),
+ ]));
+ let table = Arc::new(datafusion::datasource::empty::EmptyTable::new(Arc::clone(
+ &schema,
+ )));
+
+ let ctx = SessionContext::new();
+ ctx.register_table("empty", table)?;
+
+ let plan = ctx.table("empty").await?.into_optimized_plan()?;
+ let bytes = logical_plan_to_bytes(&plan)?;
+ let restored = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
+
+ assert_eq!(
+ format!("{}", plan.display_indent_schema()),
+ format!("{}", restored.display_indent_schema()),
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_empty_table_scan_with_projection() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new("name", DataType::Utf8, true),
+ ]));
+ let table = Arc::new(datafusion::datasource::empty::EmptyTable::new(Arc::clone(
+ &schema,
+ )));
+
+ let ctx = SessionContext::new();
+ ctx.register_table("empty", table)?;
+
+ let plan = ctx
+ .table("empty")
+ .await?
+ .select_columns(&["name"])?
+ .into_optimized_plan()?;
+ let bytes = logical_plan_to_bytes(&plan)?;
+ let restored = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
+
+ assert_eq!(
+ format!("{}", plan.display_indent_schema()),
+ format!("{}", restored.display_indent_schema()),
+ );
+ Ok(())
+}