Skip to content

Apache DataFusion SQL Query Engine

feat: Plumb Parquet virtual columns (row_number) through TableSchema and ParquetOpener (#22026)

Matt Butrovich2888bce68bc4
diff --git a/Cargo.lock b/Cargo.lock
index 66aef04c92..b0370a3e1b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2013,6 +2013,7 @@ name = "datafusion-datasource-parquet"
version = "53.1.0"
dependencies = [
"arrow",
+ "arrow-schema",
"async-trait",
"bytes",
"chrono",
diff --git a/datafusion/datasource-parquet/Cargo.toml b/datafusion/datasource-parquet/Cargo.toml
index a5855af17a..8aa6ca1f97 100644
--- a/datafusion/datasource-parquet/Cargo.toml
+++ b/datafusion/datasource-parquet/Cargo.toml
@@ -32,6 +32,7 @@ all-features = true
[dependencies]
arrow = { workspace = true }
+arrow-schema = { workspace = true }
async-trait = { workspace = true }
bytes = { workspace = true }
datafusion-common = { workspace = true, features = ["object_store", "parquet"] }
diff --git a/datafusion/datasource-parquet/src/decoder_projection.rs b/datafusion/datasource-parquet/src/decoder_projection.rs
index dcf52a37d4..27a84f2f50 100644
--- a/datafusion/datasource-parquet/src/decoder_projection.rs
+++ b/datafusion/datasource-parquet/src/decoder_projection.rs
@@ -38,10 +38,12 @@ use arrow::datatypes::SchemaRef;
use datafusion_common::Result;
use datafusion_physical_expr::projection::{ProjectionExprs, Projector};
use datafusion_physical_expr::utils::reassign_expr_columns;
+use datafusion_physical_expr_adapter::replace_columns_with_literals;
use parquet::arrow::ProjectionMask;
use parquet::schema::types::SchemaDescriptor;
+use crate::opener::{VirtualColumnsState, append_fields};
use crate::row_filter::build_projection_read_plan;
/// Per-file decoder projection: the [`ProjectionMask`] installed on every
@@ -68,19 +70,46 @@ impl DecoderProjection {
/// adapted by the per-file expr adapter); `parquet_schema` is the
/// corresponding parquet [`SchemaDescriptor`]. `output_schema` is what
/// consumers of the scan stream expect.
+ ///
+ /// `virtual_state`, when present, describes virtual columns the reader
+ /// will append to each decoded batch (e.g. parquet `row_number`). Virtual
+ /// columns are stripped from the projection fed into
+ /// `build_projection_read_plan` (which only understands file columns) and
+ /// appended to the stream schema so the projector can resolve them.
pub(crate) fn try_new(
projection: &ProjectionExprs,
physical_file_schema: &SchemaRef,
parquet_schema: &SchemaDescriptor,
output_schema: &SchemaRef,
+ virtual_state: Option<&VirtualColumnsState>,
) -> Result<Self> {
+ // Virtual columns are produced by the reader separately from the
+ // projection mask, so strip them from the expressions we feed into
+ // `build_projection_read_plan`. We substitute each virtual column
+ // reference with a null literal; that leaves the remaining Column
+ // refs (into `physical_file_schema`) intact for
+ // `ProjectionMask::roots`, which only understands file columns.
+ let projection_for_read_plan = match virtual_state {
+ None => projection.clone(),
+ Some(state) => projection.clone().try_map_exprs(|expr| {
+ replace_columns_with_literals(expr, state.null_replacements())
+ })?,
+ };
let read_plan = build_projection_read_plan(
- projection.expr_iter(),
+ projection_for_read_plan.expr_iter(),
physical_file_schema,
parquet_schema,
);
- let stream_schema = read_plan.projected_schema;
+ // The reader produces projected file columns followed by any virtual
+ // columns (`ArrowReaderOptions::with_virtual_columns` appends them to
+ // each decoded batch).
+ let stream_schema = match virtual_state {
+ Some(state) => {
+ append_fields(&read_plan.projected_schema, state.virtual_columns())
+ }
+ None => Arc::clone(&read_plan.projected_schema),
+ };
// Rebase the projection onto the decoder's stream schema (column
// indices change because the decoder yields only the masked columns).
diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rs
index cf1caf336f..bec0736366 100644
--- a/datafusion/datasource-parquet/src/mod.rs
+++ b/datafusion/datasource-parquet/src/mod.rs
@@ -43,6 +43,7 @@ pub mod source;
mod supported_predicates;
#[cfg(test)]
mod test_util;
+mod virtual_column;
mod writer;
pub use access_plan::{ParquetAccessPlan, RowGroupAccess};
@@ -60,4 +61,5 @@ pub use schema_coercion::{
transform_binary_to_string, transform_schema_to_view,
};
pub use sink::ParquetSink;
+pub use virtual_column::ParquetVirtualColumn;
pub use writer::plan_to_parquet;
diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs
index 09e7763877..c78e73119e 100644
--- a/datafusion/datasource-parquet/src/opener/mod.rs
+++ b/datafusion/datasource-parquet/src/opener/mod.rs
@@ -31,7 +31,7 @@ use crate::row_filter::RowFilterGenerator;
use crate::row_group_filter::{BloomFilterStatistics, RowGroupAccessPlanFilter};
use crate::{
Int96Coercer, ParquetAccessPlan, ParquetFileMetrics, ParquetFileReaderFactory,
- apply_file_schema_type_coercions,
+ ParquetVirtualColumn, apply_file_schema_type_coercions,
};
use arrow::array::RecordBatch;
use arrow::datatypes::DataType;
@@ -44,12 +44,16 @@ use std::future::Future;
use std::mem;
use std::sync::Arc;
-use arrow::datatypes::{SchemaRef, TimeUnit};
+use arrow::datatypes::{FieldRef, Schema, SchemaRef, TimeUnit};
#[cfg(feature = "parquet_encryption")]
use datafusion_common::encryption::FileDecryptionProperties;
use datafusion_common::stats::Precision;
-use datafusion_common::{ColumnStatistics, Result, ScalarValue, Statistics, exec_err};
+use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
+use datafusion_common::{
+ ColumnStatistics, HashSet, Result, ScalarValue, Statistics, exec_err, internal_err,
+};
use datafusion_datasource::{PartitionedFile, TableSchema};
+use datafusion_physical_expr::expressions::Column;
use datafusion_physical_expr::simplifier::PhysicalExprSimplifier;
use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
@@ -74,6 +78,152 @@ use parquet::basic::Type;
use parquet::bloom_filter::Sbbf;
use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader};
+/// Morselizer-level state for virtual columns, precomputed once per scan
+/// partition so each file skips the validator walks, `null_replacements`
+/// rebuild, and one of the `append_fields` allocations.
+///
+/// Only constructed when the scan actually requests virtual columns;
+/// [`ParquetMorselizer`] and [`PreparedParquetOpen`] hold
+/// `Option<Arc<VirtualColumnsState>>` so the zero-virtual-column path (the
+/// common case) pays nothing.
+pub(crate) struct VirtualColumnsState {
+ /// Shared list of virtual column fields. Cloned as a `Vec` only at the
+ /// arrow-rs `with_virtual_columns` call site, which takes it by value.
+ virtual_columns: Arc<Vec<FieldRef>>,
+ /// Null-literal substitutions keyed by virtual column name, used to strip
+ /// virtual-column references from the projection fed into
+ /// `build_projection_read_plan` (which only understands file columns).
+ null_replacements: HashMap<String, ScalarValue>,
+ /// `logical_file_schema` with the virtual columns appended. Fed into the
+ /// per-file expression rewriter so virtual-column references
+ /// identity-rewrite instead of being replaced with null literals.
+ logical_schema_with_virtual: SchemaRef,
+}
+
+impl VirtualColumnsState {
+ /// Validate each field carries a supported arrow virtual extension type
+ /// and precompute the per-scan derived state.
+ fn try_new(
+ virtual_columns: Vec<FieldRef>,
+ logical_file_schema: &SchemaRef,
+ ) -> Result<Self> {
+ // Gate which extension types we forward to arrow-rs. Adding a new
+ // supported virtual column means adding a `ParquetVirtualColumn`
+ // variant — not editing a stringly-typed allowlist here.
+ for field in &virtual_columns {
+ ParquetVirtualColumn::try_from(field)?;
+ }
+ let null_replacements = virtual_columns
+ .iter()
+ .map(|f| ScalarValue::try_from(f.data_type()).map(|v| (f.name().clone(), v)))
+ .collect::<Result<HashMap<String, ScalarValue>>>()?;
+ let logical_schema_with_virtual =
+ append_fields(logical_file_schema, &virtual_columns);
+ Ok(Self {
+ virtual_columns: Arc::new(virtual_columns),
+ null_replacements,
+ logical_schema_with_virtual,
+ })
+ }
+
+ /// Validated virtual column fields, in declaration order.
+ pub(crate) fn virtual_columns(&self) -> &[FieldRef] {
+ &self.virtual_columns
+ }
+
+ /// Null-literal substitutions keyed by virtual column name. Used to strip
+ /// virtual-column references from a projection before it is fed into the
+ /// parquet `ProjectionMask` (which only understands file columns).
+ pub(crate) fn null_replacements(&self) -> &HashMap<String, ScalarValue> {
+ &self.null_replacements
+ }
+}
+
+/// Build the per-scan virtual-column state.
+///
+/// Two checks run here:
+/// - Extension-type allowlist via [`VirtualColumnsState::try_new`]: returns
+/// `Err` for unsupported virtual extension types.
+/// - Predicate-reference check (when pushdown is enabled): returns `Err` if
+/// the predicate references a virtual column. The contract is that callers
+/// route filters through
+/// [`ParquetSource::try_pushdown_filters`](crate::source::ParquetSource),
+/// which classifies virtual-col filters as `PushedDown::No`. Erroring here
+/// prevents silent wrong results for callers that bypass that path and set
+/// the predicate directly on `ParquetSource`.
+///
+/// Returns `None` when the scan has no virtual columns, so callers avoid
+/// allocating the shared state on the common path.
+pub(crate) fn build_virtual_columns_state(
+ virtual_columns: &[FieldRef],
+ logical_file_schema: &SchemaRef,
+ predicate: Option<&Arc<dyn PhysicalExpr>>,
+ pushdown_filters: bool,
+) -> Result<Option<Arc<VirtualColumnsState>>> {
+ if virtual_columns.is_empty() {
+ return Ok(None);
+ }
+ if pushdown_filters && let Some(predicate) = predicate {
+ validate_predicate_does_not_reference_virtual_columns(
+ predicate,
+ virtual_columns,
+ )?;
+ }
+ let state =
+ VirtualColumnsState::try_new(virtual_columns.to_vec(), logical_file_schema)?;
+ Ok(Some(Arc::new(state)))
+}
+
+/// Return `base` unchanged when `extra` is empty; otherwise build a new schema
+/// with `extra` appended to `base`'s fields.
+pub(crate) fn append_fields(base: &SchemaRef, extra: &[FieldRef]) -> SchemaRef {
+ if extra.is_empty() {
+ return Arc::clone(base);
+ }
+ let fields = base
+ .fields()
+ .iter()
+ .cloned()
+ .chain(extra.iter().cloned())
+ .collect::<Vec<_>>();
+ Arc::new(Schema::new(fields))
+}
+
+/// Reject predicates that reference a virtual column.
+///
+/// arrow-rs's `RowFilter` evaluates predicates against a `ProjectionMask` that
+/// addresses parquet leaves only; virtual columns (e.g. `row_number`) are
+/// synthesized by the reader *after* filter evaluation and cannot be referenced
+/// inside a row filter. Silently dropping such a predicate would produce wrong
+/// results.
+fn validate_predicate_does_not_reference_virtual_columns(
+ predicate: &Arc<dyn PhysicalExpr>,
+ virtual_columns: &[FieldRef],
+) -> Result<()> {
+ if virtual_columns.is_empty() {
+ return Ok(());
+ }
+ let virtual_names: HashSet<&str> =
+ virtual_columns.iter().map(|f| f.name().as_str()).collect();
+ let mut offender: Option<String> = None;
+ predicate.apply(|node: &Arc<dyn PhysicalExpr>| {
+ if let Some(column) = node.downcast_ref::<Column>()
+ && virtual_names.contains(column.name())
+ {
+ offender = Some(column.name().to_string());
+ return Ok(TreeNodeRecursion::Stop);
+ }
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ if let Some(name) = offender {
+ return internal_err!(
+ "Predicate references virtual column '{name}'; route via \
+ ParquetSource::try_pushdown_filters."
+ );
+ }
+ Ok(())
+}
+
/// Stateless Parquet morselizer implementation.
///
/// Reading a Parquet file is a multi-stage process, with multiple CPU-intensive
@@ -139,6 +289,9 @@ pub(super) struct ParquetMorselizer {
pub reverse_row_groups: bool,
/// Optional sort order used to reorder row groups by their min/max statistics.
pub sort_order_for_reorder: Option<LexOrdering>,
+ /// Per-scan virtual-column state (validation already performed). `None`
+ /// when no virtual columns are requested — the common path.
+ pub(crate) virtual_state: Option<Arc<VirtualColumnsState>>,
}
impl fmt::Debug for ParquetMorselizer {
@@ -277,6 +430,11 @@ struct PreparedParquetOpen {
output_schema: SchemaRef,
projection: ProjectionExprs,
predicate: Option<Arc<dyn PhysicalExpr>>,
+ /// Per-scan virtual-column state, Arc-cloned from [`ParquetMorselizer`] so
+ /// each file shares validated fields, precomputed null replacements, and
+ /// the logical-with-virtual schema. `None` when no virtual columns were
+ /// requested.
+ virtual_state: Option<Arc<VirtualColumnsState>>,
reorder_predicates: bool,
pushdown_filters: bool,
force_filter_selections: bool,
@@ -649,6 +807,7 @@ impl ParquetMorselizer {
output_schema,
projection,
predicate,
+ virtual_state: self.virtual_state.as_ref().map(Arc::clone),
reorder_predicates: self.reorder_filters,
pushdown_filters: self.pushdown_filters,
force_filter_selections: self.force_filter_selections,
@@ -757,22 +916,22 @@ impl MetadataLoadedParquetOpen {
// - The logical file schema: this is the table schema minus any hive partition columns and projections.
// This is what the physical file schema is coerced to.
// - The physical file schema: this is the schema that the arrow-rs
- // parquet reader will actually produce.
+ // parquet reader will actually produce for the file's columns. Any
+ // virtual columns (see [`crate::TableSchema::virtual_columns`]) are
+ // produced separately by the reader and are not part of this schema.
let mut physical_file_schema = Arc::clone(reader_metadata.schema());
// The schema loaded from the file may not be the same as the
// desired schema (for example if we want to instruct the parquet
// reader to read strings using Utf8View instead). Update if necessary
+ let mut metadata_dirty = false;
if let Some(merged) = apply_file_schema_type_coercions(
&prepared.logical_file_schema,
&physical_file_schema,
) {
physical_file_schema = Arc::new(merged);
options = options.with_schema(Arc::clone(&physical_file_schema));
- reader_metadata = ArrowReaderMetadata::try_new(
- Arc::clone(reader_metadata.metadata()),
- options.clone(),
- )?;
+ metadata_dirty = true;
}
if let Some(ref coerce) = prepared.coerce_int96
@@ -786,6 +945,17 @@ impl MetadataLoadedParquetOpen {
{
physical_file_schema = Arc::new(merged);
options = options.with_schema(Arc::clone(&physical_file_schema));
+ metadata_dirty = true;
+ }
+
+ // Arrow-rs appends virtual columns to the supplied schema internally,
+ // so any `with_schema` coercion above must stay limited to file columns.
+ if let Some(state) = prepared.virtual_state.as_ref() {
+ options = options.with_virtual_columns((*state.virtual_columns).clone())?;
+ metadata_dirty = true;
+ }
+
+ if metadata_dirty {
reader_metadata = ArrowReaderMetadata::try_new(
Arc::clone(reader_metadata.metadata()),
options.clone(),
@@ -808,11 +978,32 @@ impl MetadataLoadedParquetOpen {
let needs_rewrite = prepared.predicate.is_some()
|| prepared.logical_file_schema != physical_file_schema;
if needs_rewrite {
+ // When virtual columns are requested, augment the logical and
+ // physical schemas passed to the rewriter/simplifier with those
+ // fields. The rewriter identity-rewrites references found in both
+ // schemas, keeping virtual-column references as `Column` rather
+ // than replacing them with null literals; the simplifier needs
+ // them present so it can resolve their data types while walking
+ // expression trees. We keep `physical_file_schema` itself as the
+ // pure file schema so downstream predicate pushdown, pruning, and
+ // row filter construction stay unaffected.
+ let (logical_for_rewrite, physical_for_rewrite) =
+ if let Some(state) = prepared.virtual_state.as_ref() {
+ (
+ Arc::clone(&state.logical_schema_with_virtual),
+ append_fields(&physical_file_schema, &state.virtual_columns),
+ )
+ } else {
+ (
+ Arc::clone(&prepared.logical_file_schema),
+ Arc::clone(&physical_file_schema),
+ )
+ };
let rewriter = prepared.expr_adapter_factory.create(
- Arc::clone(&prepared.logical_file_schema),
- Arc::clone(&physical_file_schema),
+ Arc::clone(&logical_for_rewrite),
+ Arc::clone(&physical_for_rewrite),
)?;
- let simplifier = PhysicalExprSimplifier::new(&physical_file_schema);
+ let simplifier = PhysicalExprSimplifier::new(&physical_for_rewrite);
prepared.predicate = prepared
.predicate
.map(|p| simplifier.simplify(rewriter.rewrite(p)?))
@@ -1156,6 +1347,7 @@ impl RowGroupsPrunedParquetOpen {
&prepared.physical_file_schema,
reader_metadata.parquet_schema(),
&prepared.output_schema,
+ prepared.virtual_state.as_deref(),
)?;
let (decoder, pending_decoders, remaining_limit) = {
@@ -1505,12 +1697,28 @@ mod test {
self
}
- /// Set projection by column indices (convenience method for common case).
+ /// Set projection by column indices.
+ ///
+ /// The indices are resolved against the **file schema**, not the full
+ /// table schema. Callers that need to project partition columns or
+ /// virtual columns must use [`Self::with_projection`] and construct a
+ /// [`ProjectionExprs`] against [`TableSchema::table_schema`].
fn with_projection_indices(mut self, indices: &[usize]) -> Self {
self.projection_indices = Some(indices.to_vec());
self
}
+ /// Set an explicit projection.
+ ///
+ /// Prefer this over [`Self::with_projection_indices`] whenever the
+ /// projection must reference partition or virtual columns, since
+ /// `with_projection_indices` resolves its indices against the file
+ /// schema only.
+ fn with_projection(mut self, projection: ProjectionExprs) -> Self {
+ self.projection = Some(projection);
+ self
+ }
+
/// Set the predicate.
fn with_predicate(mut self, predicate: Arc<dyn PhysicalExpr>) -> Self {
self.predicate = Some(predicate);
@@ -1553,12 +1761,26 @@ mod test {
self
}
- /// Build the ParquetMorselizer instance.
+ /// Build the ParquetMorselizer instance, unwrapping validation errors.
///
/// # Panics
///
- /// Panics if required fields (store, schema/table_schema) are not set.
+ /// Panics if required fields (store, schema/table_schema) are not set,
+ /// or if virtual-column validation fails. Use [`Self::try_build`]
+ /// when the test wants to assert on the validation error.
fn build(self) -> ParquetMorselizer {
+ self.try_build().expect("ParquetMorselizerBuilder::build")
+ }
+
+ /// Build the ParquetMorselizer instance, returning any morselizer-level
+ /// validation error (e.g. unsupported virtual extension type, or a
+ /// predicate that references a virtual column with
+ /// `pushdown_filters=true`).
+ ///
+ /// # Panics
+ ///
+ /// Panics if required fields (store, schema/table_schema) are not set.
+ fn try_build(self) -> Result<ParquetMorselizer> {
let store = self
.store
.expect("ParquetMorselizerBuilder: store must be set via with_store()");
@@ -1577,7 +1799,14 @@ mod test {
ProjectionExprs::from_indices(&all_indices, &file_schema)
};
- ParquetMorselizer {
+ let virtual_state = build_virtual_columns_state(
+ table_schema.virtual_columns(),
+ table_schema.file_schema(),
+ self.predicate.as_ref(),
+ self.pushdown_filters,
+ )?;
+
+ Ok(ParquetMorselizer {
partition_index: self.partition_index,
projection,
batch_size: self.batch_size,
@@ -1609,7 +1838,8 @@ mod test {
max_predicate_cache_size: self.max_predicate_cache_size,
reverse_row_groups: self.reverse_row_groups,
sort_order_for_reorder: None,
- }
+ virtual_state,
+ })
}
}
@@ -1770,7 +2000,7 @@ mod test {
async fn write_parquet(
store: Arc<dyn ObjectStore>,
filename: &str,
- batch: arrow::record_batch::RecordBatch,
+ batch: RecordBatch,
) -> usize {
write_parquet_batches(store, filename, vec![batch], None).await
}
@@ -1779,7 +2009,7 @@ mod test {
async fn write_parquet_batches(
store: Arc<dyn ObjectStore>,
filename: &str,
- batches: Vec<arrow::record_batch::RecordBatch>,
+ batches: Vec<RecordBatch>,
props: Option<WriterProperties>,
) -> usize {
let mut out = BytesMut::new().writer();
@@ -2683,4 +2913,433 @@ mod test {
assert!(runs[2].needs_filter);
assert_eq!(runs[2].access_plan.row_group_indexes(), vec![3]);
}
+
+ /// Helpers for tests that exercise parquet virtual columns
+ /// (e.g. `row_number`) plumbed through `TableSchema`/`ParquetOpener`.
+ mod virtual_columns {
+ use super::*;
+ use arrow::array::{Array, Int64Array};
+ use arrow::datatypes::FieldRef;
+ use parquet::arrow::RowNumber;
+
+ /// Build a parquet `row_number` virtual column field. Spark's
+ /// `_tmp_metadata_row_index` is declared nullable, so the default
+ /// matches that contract; tests that need `nullable=false` can
+ /// override via `with_nullable`.
+ fn row_number_field(name: &str, nullable: bool) -> FieldRef {
+ Arc::new(
+ Field::new(name, DataType::Int64, nullable)
+ .with_extension_type(RowNumber),
+ )
+ }
+
+ /// Collect every `Int64` value from the given column in every batch
+ /// of a stream. Used to verify the `row_number` column end to end.
+ async fn collect_int64_values(
+ mut stream: BoxStream<'static, Result<RecordBatch>>,
+ column: usize,
+ ) -> Vec<i64> {
+ let mut out = vec![];
+ while let Some(batch) = stream.next().await {
+ let batch = batch.unwrap();
+ let array = batch
+ .column(column)
+ .as_any()
+ .downcast_ref::<Int64Array>()
+ .expect("expected Int64 column");
+ for i in 0..array.len() {
+ assert!(
+ !array.is_null(i),
+ "row_number values produced by the reader must not be null"
+ );
+ out.push(array.value(i));
+ }
+ }
+ out
+ }
+
+ /// Write a parquet file containing `num_row_groups` groups of
+ /// `rows_per_group` rows with a single `value` Int64 column.
+ /// Values are `0..num_row_groups*rows_per_group`.
+ async fn write_grouped_file(
+ store: &Arc<dyn ObjectStore>,
+ path: &str,
+ num_row_groups: usize,
+ rows_per_group: usize,
+ ) -> (SchemaRef, usize) {
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "value",
+ DataType::Int64,
+ false,
+ )]));
+ let mut batches = Vec::with_capacity(num_row_groups);
+ for g in 0..num_row_groups {
+ let start = (g * rows_per_group) as i64;
+ let values: Vec<i64> = (start..start + rows_per_group as i64).collect();
+ batches.push(
+ RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(Int64Array::from(values))],
+ )
+ .unwrap(),
+ );
+ }
+ let props = WriterProperties::builder()
+ .set_max_row_group_row_count(Some(rows_per_group))
+ .build();
+ let data_size =
+ write_parquet_batches(Arc::clone(store), path, batches, Some(props))
+ .await;
+ (schema, data_size)
+ }
+
+ #[tokio::test]
+ async fn test_row_index_basic() {
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let (file_schema, data_size) =
+ write_grouped_file(&store, "basic.parquet", 1, 5).await;
+
+ let rn_field = row_number_field("row_number", false);
+ let table_schema = TableSchemaBuilder::new(Arc::clone(&file_schema))
+ .with_virtual_columns(vec![Arc::clone(&rn_field)])
+ .build();
+ // Project [value, row_number] — indices in table_schema are
+ // [0 file:value, 1 virtual:row_number].
+ let projection =
+ ProjectionExprs::from_indices(&[0, 1], table_schema.table_schema());
+
+ let morselizer = ParquetMorselizerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_table_schema(table_schema)
+ .with_projection(projection)
+ .build();
+
+ let file = PartitionedFile::new(
+ "basic.parquet".to_string(),
+ u64::try_from(data_size).unwrap(),
+ );
+ let stream = open_file(&morselizer, file).await.unwrap();
+ let row_numbers = collect_int64_values(stream, 1).await;
+ assert_eq!(row_numbers, vec![0, 1, 2, 3, 4]);
+ }
+
+ #[tokio::test]
+ async fn test_row_index_projection_only() {
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let (file_schema, data_size) =
+ write_grouped_file(&store, "proj_only.parquet", 1, 4).await;
+
+ let rn_field = row_number_field("row_number", false);
+ let table_schema = TableSchemaBuilder::new(Arc::clone(&file_schema))
+ .with_virtual_columns(vec![Arc::clone(&rn_field)])
+ .build();
+ // Project only the virtual column (index 1).
+ let projection =
+ ProjectionExprs::from_indices(&[1], table_schema.table_schema());
+
+ let morselizer = ParquetMorselizerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_table_schema(table_schema)
+ .with_projection(projection)
+ .build();
+
+ let file = PartitionedFile::new(
+ "proj_only.parquet".to_string(),
+ u64::try_from(data_size).unwrap(),
+ );
+ let stream = open_file(&morselizer, file).await.unwrap();
+ let row_numbers = collect_int64_values(stream, 0).await;
+ assert_eq!(row_numbers, vec![0, 1, 2, 3]);
+ }
+
+ #[tokio::test]
+ async fn test_row_index_multi_row_group() {
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let (file_schema, data_size) =
+ write_grouped_file(&store, "multi_rg.parquet", 3, 100).await;
+
+ let rn_field = row_number_field("row_number", false);
+ let table_schema = TableSchemaBuilder::new(Arc::clone(&file_schema))
+ .with_virtual_columns(vec![Arc::clone(&rn_field)])
+ .build();
+ let projection =
+ ProjectionExprs::from_indices(&[0, 1], table_schema.table_schema());
+
+ let morselizer = ParquetMorselizerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_table_schema(table_schema)
+ .with_projection(projection)
+ .build();
+
+ let file = PartitionedFile::new(
+ "multi_rg.parquet".to_string(),
+ u64::try_from(data_size).unwrap(),
+ );
+ let stream = open_file(&morselizer, file).await.unwrap();
+ let row_numbers = collect_int64_values(stream, 1).await;
+ let expected: Vec<i64> = (0..300).collect();
+ assert_eq!(row_numbers, expected);
+ }
+
+ #[tokio::test]
+ async fn test_row_index_with_row_group_skip() {
+ // 3 row groups of 100 rows. A predicate that excludes the middle
+ // row group (values 100..200) must leave absolute row numbers
+ // 0..100 and 200..300 intact — not 0..200. This guards against
+ // the arrow-rs bug fixed in apache/arrow-rs#8863.
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let (file_schema, data_size) =
+ write_grouped_file(&store, "rg_skip.parquet", 3, 100).await;
+
+ let rn_field = row_number_field("row_number", false);
+ let table_schema = TableSchemaBuilder::new(Arc::clone(&file_schema))
+ .with_virtual_columns(vec![Arc::clone(&rn_field)])
+ .build();
+ let projection =
+ ProjectionExprs::from_indices(&[0, 1], table_schema.table_schema());
+
+ // `value < 100 OR value >= 200` prunes the middle row group via
+ // min/max statistics.
+ let expr = col("value")
+ .lt(lit(100i64))
+ .or(col("value").gt_eq(lit(200i64)));
+ let predicate = logical2physical(&expr, table_schema.table_schema());
+
+ let morselizer = ParquetMorselizerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_table_schema(table_schema)
+ .with_projection(projection)
+ .with_predicate(predicate)
+ .with_row_group_stats_pruning(true)
+ .build();
+
+ let file = PartitionedFile::new(
+ "rg_skip.parquet".to_string(),
+ u64::try_from(data_size).unwrap(),
+ );
+ let stream = open_file(&morselizer, file).await.unwrap();
+ let row_numbers = collect_int64_values(stream, 1).await;
+ let expected: Vec<i64> = (0..100).chain(200..300).collect();
+ assert_eq!(row_numbers, expected);
+ }
+
+ #[tokio::test]
+ async fn test_row_index_with_partition_cols() {
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let (file_schema, data_size) =
+ write_grouped_file(&store, "part=5/data.parquet", 1, 3).await;
+
+ let rn_field = row_number_field("row_number", false);
+ let partition_col = Arc::new(Field::new("part", DataType::Int32, false));
+ let table_schema = TableSchemaBuilder::new(Arc::clone(&file_schema))
+ .with_table_partition_cols(vec![Arc::clone(&partition_col)])
+ .with_virtual_columns(vec![Arc::clone(&rn_field)])
+ .build();
+ // table_schema layout: [value(0), part(1), row_number(2)].
+ let projection =
+ ProjectionExprs::from_indices(&[0, 1, 2], table_schema.table_schema());
+
+ let morselizer = ParquetMorselizerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_table_schema(table_schema)
+ .with_projection(projection)
+ .build();
+
+ let mut file = PartitionedFile::new(
+ "part=5/data.parquet".to_string(),
+ u64::try_from(data_size).unwrap(),
+ );
+ file.partition_values = vec![ScalarValue::Int32(Some(5))];
+
+ let stream = open_file(&morselizer, file).await.unwrap();
+ let mut stream = stream;
+ let batch = stream.next().await.unwrap().unwrap();
+ assert!(stream.next().await.is_none());
+
+ assert_eq!(batch.num_columns(), 3);
+ assert_eq!(batch.schema().field(0).name(), "value");
+ assert_eq!(batch.schema().field(1).name(), "part");
+ assert_eq!(batch.schema().field(2).name(), "row_number");
+
+ let part = batch
+ .column(1)
+ .as_any()
+ .downcast_ref::<arrow::array::Int32Array>()
+ .unwrap();
+ assert!(part.iter().all(|v| v == Some(5)));
+
+ let rn = batch
+ .column(2)
+ .as_any()
+ .downcast_ref::<Int64Array>()
+ .unwrap();
+ let rn_values: Vec<i64> = (0..rn.len()).map(|i| rn.value(i)).collect();
+ assert_eq!(rn_values, vec![0, 1, 2]);
+ }
+
+ #[tokio::test]
+ async fn test_row_index_nullable_int64() {
+ // Spark declares `_tmp_metadata_row_index` nullable. Verify the
+ // nullability flag flows through unchanged.
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let (file_schema, data_size) =
+ write_grouped_file(&store, "nullable.parquet", 1, 3).await;
+
+ let rn_field = row_number_field("_tmp_metadata_row_index", true);
+ let table_schema = TableSchemaBuilder::new(Arc::clone(&file_schema))
+ .with_virtual_columns(vec![Arc::clone(&rn_field)])
+ .build();
+ let projection =
+ ProjectionExprs::from_indices(&[0, 1], table_schema.table_schema());
+
+ let morselizer = ParquetMorselizerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_table_schema(table_schema)
+ .with_projection(projection)
+ .build();
+
+ let file = PartitionedFile::new(
+ "nullable.parquet".to_string(),
+ u64::try_from(data_size).unwrap(),
+ );
+ let mut stream = open_file(&morselizer, file).await.unwrap();
+ let batch = stream.next().await.unwrap().unwrap();
+
+ let schema_field = batch.schema().field(1).clone();
+ assert_eq!(schema_field.name(), "_tmp_metadata_row_index");
+ assert_eq!(schema_field.data_type(), &DataType::Int64);
+ assert!(
+ schema_field.is_nullable(),
+ "nullable flag should be preserved for Spark's row index field"
+ );
+ }
+
+ #[tokio::test]
+ async fn test_unsupported_virtual_extension_type_rejected() {
+ // Guard: opener must reject virtual columns carrying extension
+ // types outside the tested allowlist, rather than silently
+ // forwarding them to arrow-rs (where they would produce columns
+ // we have not validated against DataFusion's projection and
+ // predicate paths).
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let (file_schema, _data_size) =
+ write_grouped_file(&store, "unsupported.parquet", 1, 1).await;
+
+ // RowGroupIndex is a real arrow-rs virtual type but is not in
+ // SUPPORTED_VIRTUAL_EXTENSION_TYPES until a test is added for it.
+ let rg_field = Arc::new(
+ Field::new("row_group_index", DataType::Int64, false)
+ .with_extension_type(parquet::arrow::RowGroupIndex),
+ );
+ let table_schema = TableSchemaBuilder::new(Arc::clone(&file_schema))
+ .with_virtual_columns(vec![rg_field])
+ .build();
+ let projection =
+ ProjectionExprs::from_indices(&[0, 1], table_schema.table_schema());
+
+ // Validation now happens at morselizer-build time (once per scan
+ // partition), not once per file inside `prepare_open_file`.
+ let err = ParquetMorselizerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_table_schema(table_schema)
+ .with_projection(projection)
+ .try_build()
+ .unwrap_err();
+ let msg = err.to_string();
+ assert!(
+ msg.contains("parquet.virtual.row_group_index"),
+ "error should name the unsupported extension type, got: {msg}"
+ );
+ }
+
+ /// Build a morselizer + file for a 5-row single-row-group parquet at
+ /// `path`, with a single `row_number` virtual column and the given
+ /// physical predicate applied to
+ /// `table_schema = [value(0), row_number(1)]`.
+ async fn build_pushdown_morselizer(
+ store: &Arc<dyn ObjectStore>,
+ path: &str,
+ predicate_expr: datafusion_expr::Expr,
+ pushdown_filters: bool,
+ ) -> Result<(ParquetMorselizer, PartitionedFile)> {
+ let (file_schema, data_size) = write_grouped_file(store, path, 1, 5).await;
+ let rn_field = row_number_field("row_number", false);
+ let table_schema = TableSchemaBuilder::new(Arc::clone(&file_schema))
+ .with_virtual_columns(vec![Arc::clone(&rn_field)])
+ .build();
+ let projection =
+ ProjectionExprs::from_indices(&[0, 1], table_schema.table_schema());
+ let predicate =
+ logical2physical(&predicate_expr, table_schema.table_schema());
+
+ let morselizer = ParquetMorselizerBuilder::new()
+ .with_store(Arc::clone(store))
+ .with_table_schema(table_schema)
+ .with_projection(projection)
+ .with_predicate(predicate)
+ .with_pushdown_filters(pushdown_filters)
+ .try_build()?;
+
+ let file =
+ PartitionedFile::new(path.to_string(), u64::try_from(data_size).unwrap());
+ Ok((morselizer, file))
+ }
+
+ // The predicate-vs-virtual-column check rejects callers that bypass
+ // `ParquetSource::try_pushdown_filters` (which keeps virtual-col
+ // filters above the scan as a `FilterExec`) and set the predicate
+ // directly on the source with pushdown enabled. Without this guard,
+ // arrow-rs's `RowFilter` would silently drop the virtual-col conjunct
+ // and produce wrong results.
+ #[tokio::test]
+ async fn test_row_index_predicate_pushdown_mixed_or_errors() {
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let expr = col("row_number")
+ .eq(lit(2i64))
+ .or(col("value").eq(lit(4i64)));
+ let err =
+ build_pushdown_morselizer(&store, "pushdown_mixed.parquet", expr, true)
+ .await
+ .unwrap_err();
+ assert!(
+ err.to_string().contains("try_pushdown_filters"),
+ "error should mention try_pushdown_filters, got: {err}"
+ );
+ }
+
+ #[tokio::test]
+ async fn test_row_index_predicate_pushdown_virtual_only_errors() {
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let expr = col("row_number").eq(lit(2i64));
+ let err = build_pushdown_morselizer(
+ &store,
+ "pushdown_virtual_only.parquet",
+ expr,
+ true,
+ )
+ .await
+ .unwrap_err();
+ assert!(
+ err.to_string().contains("try_pushdown_filters"),
+ "error should mention try_pushdown_filters, got: {err}"
+ );
+ }
+
+ #[tokio::test]
+ async fn test_row_index_predicate_allowed_when_pushdown_disabled() {
+ // Guards the `pushdown_filters=false` path: the predicate is only
+ // used for stats pruning (a no-op for row_number) and must not
+ // trip the virtual-column check.
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let expr = col("row_number").eq(lit(2i64));
+ let (morselizer, file) =
+ build_pushdown_morselizer(&store, "pushdown_off.parquet", expr, false)
+ .await
+ .unwrap();
+
+ let stream = open_file(&morselizer, file).await.unwrap();
+ let (_batches, rows) = count_batches_and_rows(stream).await;
+ assert_eq!(rows, 5);
+ }
+ }
}
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs
index 8952666491..acba8ff828 100644
--- a/datafusion/datasource-parquet/src/source.rs
+++ b/datafusion/datasource-parquet/src/source.rs
@@ -24,6 +24,7 @@ use crate::DefaultParquetFileReaderFactory;
use crate::ParquetFileReaderFactory;
use crate::opener::ParquetMorselizer;
use crate::opener::build_pruning_predicates;
+use crate::opener::build_virtual_columns_state;
use crate::row_filter::can_expr_be_pushed_down_with_schemas;
use datafusion_common::config::ConfigOptions;
#[cfg(feature = "parquet_encryption")]
@@ -346,7 +347,11 @@ impl ParquetSource {
self
}
- /// Set predicate information
+ /// Set predicate information.
+ ///
+ /// Predicates referencing virtual columns must go through
+ /// [`Self::try_pushdown_filters`]. Passing them here with pushdown
+ /// enabled trips a debug assert in the opener.
#[expect(clippy::needless_pass_by_value)]
pub fn with_predicate(&self, predicate: Arc<dyn PhysicalExpr>) -> Self {
let mut conf = self.clone();
@@ -584,6 +589,22 @@ impl FileSource for ParquetSource {
);
}
+ // Validate virtual columns (extension-type allowlist) and, when
+ // pushdown is enabled, reject predicates that reference them. Both
+ // checks depend only on morselizer-level state, so we pay their cost
+ // once per scan partition rather than per file.
+ //
+ // Gating predicate validation on `pushdown_filters` is deliberate:
+ // when pushdown is off the predicate stays above the scan as a
+ // `FilterExec` and resolves virtual columns there; the row-filter
+ // ban only applies to the pushdown path.
+ let virtual_state = build_virtual_columns_state(
+ self.table_schema.virtual_columns(),
+ self.table_schema.file_schema(),
+ self.predicate.as_ref(),
+ self.pushdown_filters(),
+ )?;
+
Ok(Box::new(ParquetMorselizer {
partition_index: partition,
projection: self.projection.clone(),
@@ -613,6 +634,7 @@ impl FileSource for ParquetSource {
max_predicate_cache_size: self.max_predicate_cache_size(),
reverse_row_groups: self.reverse_row_groups,
sort_order_for_reorder: self.sort_order_for_reorder.clone(),
+ virtual_state,
}))
}
@@ -727,7 +749,12 @@ impl FileSource for ParquetSource {
filters: Vec<Arc<dyn PhysicalExpr>>,
config: &ConfigOptions,
) -> datafusion_common::Result<FilterPushdownPropagation<Arc<dyn FileSource>>> {
- let table_schema = self.table_schema.table_schema();
+ // Use the schema excluding virtual columns: virtual columns (e.g.
+ // Parquet `row_number`) are produced by the reader itself and cannot
+ // be referenced inside a RowFilter, so predicates that reference them
+ // must not be marked as pushed down — otherwise the scan would
+ // silently drop them and produce wrong results.
+ let pushable_schema = self.table_schema.schema_without_virtual_columns();
// Determine if based on configs we should push filters down.
// If either the table / scan itself or the config has pushdown enabled,
// we will push down the filters.
@@ -743,7 +770,7 @@ impl FileSource for ParquetSource {
let filters: Vec<PushedDownPredicate> = filters
.into_iter()
.map(|filter| {
- if can_expr_be_pushed_down_with_schemas(&filter, table_schema) {
+ if can_expr_be_pushed_down_with_schemas(&filter, &pushable_schema) {
PushedDownPredicate::supported(filter)
} else {
PushedDownPredicate::unsupported(filter)
@@ -1583,4 +1610,67 @@ mod tests {
);
}
}
+
+ #[test]
+ fn test_try_pushdown_filters_rejects_virtual_column_refs() {
+ // Virtual columns are produced by the reader and cannot be referenced
+ // inside a RowFilter. `try_pushdown_filters` must report such filters
+ // as `PushedDown::No` so the FilterExec above the scan stays in
+ // place — otherwise the scan would silently drop the predicate and
+ // produce wrong results.
+ use arrow::datatypes::{DataType, Field, FieldRef, Schema};
+ use datafusion_common::config::ConfigOptions;
+ use datafusion_datasource::TableSchema;
+ use datafusion_expr::{col, lit as logical_lit};
+ use datafusion_physical_expr::planner::logical2physical;
+ use datafusion_physical_plan::filter_pushdown::PushedDown;
+ use parquet::arrow::RowNumber;
+
+ let file_schema = Arc::new(Schema::new(vec![Field::new(
+ "value",
+ DataType::Int64,
+ false,
+ )]));
+ let row_number_field: FieldRef = Arc::new(
+ Field::new("row_number", DataType::Int64, false)
+ .with_extension_type(RowNumber),
+ );
+ let table_schema = TableSchema::builder(file_schema)
+ .with_virtual_columns(vec![row_number_field])
+ .build();
+
+ let source = ParquetSource::new(table_schema).with_pushdown_filters(true);
+
+ let full_schema = source.table_schema.table_schema();
+
+ let pushable = logical2physical(&col("value").eq(logical_lit(1i64)), full_schema);
+ let virtual_only =
+ logical2physical(&col("row_number").eq(logical_lit(2i64)), full_schema);
+ let mixed = logical2physical(
+ &col("row_number")
+ .eq(logical_lit(2i64))
+ .or(col("value").eq(logical_lit(4i64))),
+ full_schema,
+ );
+
+ let config = ConfigOptions::default();
+ let prop = source
+ .try_pushdown_filters(vec![pushable, virtual_only, mixed], &config)
+ .expect("try_pushdown_filters must not error");
+
+ assert_eq!(prop.filters.len(), 3);
+ assert!(
+ matches!(prop.filters[0], PushedDown::Yes),
+ "file-column filter should be pushable"
+ );
+ assert!(
+ matches!(prop.filters[1], PushedDown::No),
+ "filter referencing only a virtual column must not be pushed down"
+ );
+ assert!(
+ matches!(prop.filters[2], PushedDown::No),
+ "filter mixing a virtual column with a file column must not be \
+ pushed down (row filter would silently drop it)"
+ );
+ }
}
diff --git a/datafusion/datasource-parquet/src/virtual_column.rs b/datafusion/datasource-parquet/src/virtual_column.rs
new file mode 100644
index 0000000000..2290ad2aea
--- /dev/null
+++ b/datafusion/datasource-parquet/src/virtual_column.rs
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Typed wrapper for parquet virtual columns.
+//!
+//! arrow-rs identifies virtual columns via arrow extension types carried on
+//! the `FieldRef`. [`ParquetVirtualColumn`] lifts that contract into the type
+//! system so callers validate at the boundary (via `TryFrom<&FieldRef>`)
+//! rather than string-comparing extension-type names deep inside the reader.
+
+use arrow::datatypes::FieldRef;
+use arrow_schema::extension::ExtensionType;
+use datafusion_common::{DataFusionError, Result, not_impl_err};
+use parquet::arrow::RowNumber;
+use std::sync::Arc;
+
+/// A parquet virtual column validated to have a supported arrow extension
+/// type.
+///
+/// Construct via [`TryFrom<&FieldRef>`]; add a new variant (and update the
+/// `TryFrom` impl) when DataFusion gains support for another arrow-rs virtual
+/// extension type.
+#[derive(Debug, Clone)]
+pub enum ParquetVirtualColumn {
+ /// Absolute row number within the parquet file. Backed by arrow-rs's
+ /// [`RowNumber`] extension type.
+ RowNumber(FieldRef),
+}
+
+impl ParquetVirtualColumn {
+ pub fn field(&self) -> &FieldRef {
+ match self {
+ Self::RowNumber(field) => field,
+ }
+ }
+}
+
+impl From<ParquetVirtualColumn> for FieldRef {
+ fn from(col: ParquetVirtualColumn) -> Self {
+ match col {
+ ParquetVirtualColumn::RowNumber(field) => field,
+ }
+ }
+}
+
+impl TryFrom<&FieldRef> for ParquetVirtualColumn {
+ type Error = DataFusionError;
+
+ fn try_from(field: &FieldRef) -> Result<Self> {
+ let Some(name) = field.extension_type_name() else {
+ return not_impl_err!(
+ "Virtual column '{}' is missing an Arrow extension type; \
+ supported extension types: [{}]",
+ field.name(),
+ RowNumber::NAME
+ );
+ };
+ match name {
+ n if n == RowNumber::NAME => Ok(Self::RowNumber(Arc::clone(field))),
+ other => not_impl_err!(
+ "Virtual column '{}' uses unsupported Arrow extension type '{}'; \
+ supported types: [{}]. Add a ParquetVirtualColumn variant and \
+ a test for this type before wiring it through.",
+ field.name(),
+ other,
+ RowNumber::NAME
+ ),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::datatypes::{DataType, Field};
+
+ #[test]
+ fn row_number_field_converts() {
+ let field: FieldRef = Arc::new(
+ Field::new("row_number", DataType::Int64, false)
+ .with_extension_type(RowNumber),
+ );
+ let col = ParquetVirtualColumn::try_from(&field).expect("valid row_number");
+ assert!(matches!(col, ParquetVirtualColumn::RowNumber(_)));
+ assert_eq!(col.field().name(), "row_number");
+ }
+
+ #[test]
+ fn missing_extension_type_rejected() {
+ let field: FieldRef = Arc::new(Field::new("plain", DataType::Int64, false));
+ let err = ParquetVirtualColumn::try_from(&field).unwrap_err();
+ assert!(
+ err.to_string().contains("missing an Arrow extension type"),
+ "got: {err}"
+ );
+ }
+
+ #[test]
+ fn unsupported_extension_type_rejected() {
+ // RowGroupIndex is a real arrow-rs virtual type not yet in our enum.
+ let field: FieldRef = Arc::new(
+ Field::new("row_group_index", DataType::Int64, false)
+ .with_extension_type(parquet::arrow::RowGroupIndex),
+ );
+ let err = ParquetVirtualColumn::try_from(&field).unwrap_err();
+ assert!(
+ err.to_string().contains("parquet.virtual.row_group_index"),
+ "error should name the offending extension type, got: {err}"
+ );
+ }
+}
diff --git a/datafusion/datasource/src/table_schema.rs b/datafusion/datasource/src/table_schema.rs
index 8b6d18b0e5..085040e7de 100644
--- a/datafusion/datasource/src/table_schema.rs
+++ b/datafusion/datasource/src/table_schema.rs
@@ -23,10 +23,17 @@ use std::sync::Arc;
/// The overall schema for potentially partitioned data sources.
///
/// When reading partitioned data (such as Hive-style partitioning), a [`TableSchema`]
-/// consists of two parts:
+/// consists of up to three parts:
/// 1. **File schema**: The schema of the actual data files on disk
/// 2. **Partition columns**: Columns whose values are encoded in the directory structure,
/// but not stored in the files themselves
+/// 3. **Virtual columns**: Columns produced by the file reader (e.g. Parquet
+/// `row_number`) that are not stored in the files
+///
+/// The full table schema is composed in that order: file columns, then
+/// partition columns, then virtual columns. Consumers that need a different
+/// output ordering should use a projection on top of
+/// [`TableSchema::table_schema`].
///
/// # Example: Partitioned Table
///
@@ -76,10 +83,24 @@ pub struct TableSchema {
/// with an existing schema.
table_partition_cols: Fields,
- /// The complete table schema: file_schema columns followed by partition columns.
+ /// Virtual columns that are generated by the reader rather than read from
+ /// the data files or the directory structure.
+ ///
+ /// For example, a Parquet reader may inject a `row_number` column whose
+ /// values are produced per file by the reader. Virtual column fields must
+ /// carry an arrow extension type (e.g. `RowNumber`, `RowGroupIndex`) so the
+ /// file reader can recognize them.
///
- /// This is pre-computed during construction by concatenating `file_schema`
- /// and `table_partition_cols`, so it can be returned as a cheap reference.
+ /// Virtual columns are appended at the end of the table schema, after the
+ /// file columns and any partition columns (layout: `[file, partition,
+ /// virtual]`).
+ virtual_columns: Fields,
+
+ /// The complete table schema: file_schema columns, followed by partition
+ /// columns, followed by virtual columns.
+ ///
+ /// This is pre-computed during construction by concatenating the three
+ /// parts, so it can be returned as a cheap reference.
table_schema: SchemaRef,
}
@@ -140,7 +161,7 @@ impl TableSchema {
}
/// Return a new `TableSchema` with `partition_cols` as its partition columns,
- /// replacing any existing ones.
+ /// replacing any existing ones. Existing virtual columns are preserved.
#[deprecated(
since = "55.0.0",
note = "use TableSchema::builder(file_schema).with_table_partition_cols(cols).build()"
@@ -148,6 +169,22 @@ impl TableSchema {
pub fn with_table_partition_cols(self, partition_cols: Vec<FieldRef>) -> Self {
TableSchemaBuilder::new(self.file_schema)
.with_table_partition_cols(partition_cols)
+ .with_virtual_columns(self.virtual_columns)
+ .build()
+ }
+
+ /// Return a new `TableSchema` with `virtual_columns` as its virtual columns,
+ /// replacing any existing ones. Existing partition columns are preserved.
+ ///
+ /// Virtual columns are produced by the file reader (e.g. a Parquet
+ /// `row_number` column) rather than stored in the files or derived from
+ /// partition paths. Each field must carry an arrow virtual extension type so
+ /// the reader can recognize it; `ParquetOpener` forwards these fields to
+ /// `parquet::arrow::arrow_reader::ArrowReaderOptions::with_virtual_columns`.
+ pub fn with_virtual_columns(self, virtual_columns: Vec<FieldRef>) -> Self {
+ TableSchemaBuilder::new(self.file_schema)
+ .with_table_partition_cols(self.table_partition_cols)
+ .with_virtual_columns(virtual_columns)
.build()
}
@@ -166,13 +203,43 @@ impl TableSchema {
&self.table_partition_cols
}
- /// Get the full table schema (file schema + partition columns).
+ /// Get the virtual columns.
///
- /// This is the complete schema that will be seen by queries, combining
- /// both the columns from the files and the partition columns.
+ /// Virtual columns are produced by the file reader (e.g. Parquet
+ /// `row_number`) and are not stored in the data files or derived from
+ /// partition paths.
+ pub fn virtual_columns(&self) -> &Fields {
+ &self.virtual_columns
+ }
+
+ /// Get the full table schema (file schema + partition columns + virtual columns).
+ ///
+ /// This is the complete schema that will be seen by queries. Fields appear
+ /// in the order: file columns, partition columns, virtual columns.
pub fn table_schema(&self) -> &SchemaRef {
&self.table_schema
}
+
+ /// Schema of columns that can be referenced by predicates pushed into the
+ /// file reader: file columns plus partition columns, excluding virtual
+ /// columns.
+ ///
+ /// Virtual columns are produced by the reader itself (e.g. Parquet
+ /// `row_number`) and cannot be referenced inside the reader's row filter,
+ /// so predicates that reference them must stay above the scan. Callers
+ /// deciding which filters to push down should check against this schema
+ /// rather than [`Self::table_schema`].
+ ///
+ /// When there are no virtual columns this returns the same schema as
+ /// [`Self::table_schema`].
+ pub fn schema_without_virtual_columns(&self) -> SchemaRef {
+ if self.virtual_columns.is_empty() {
+ return Arc::clone(&self.table_schema);
+ }
+ let mut builder = SchemaBuilder::from(self.file_schema.as_ref());
+ builder.extend(self.table_partition_cols.iter().cloned());
+ Arc::new(builder.finish())
+ }
}
impl From<SchemaRef> for TableSchema {
@@ -189,9 +256,10 @@ impl From<&SchemaRef> for TableSchema {
/// Builder for [`TableSchema`].
///
-/// The file schema is the only required input; partition columns are optional.
-/// Unlike calling [`TableSchema`]'s setters repeatedly, the builder computes the
-/// concatenated table schema exactly once, in [`TableSchemaBuilder::build`].
+/// The file schema is the only required input; partition columns and virtual
+/// columns are optional. Unlike calling [`TableSchema`]'s setters repeatedly,
+/// the builder computes the concatenated table schema exactly once, in
+/// [`TableSchemaBuilder::build`].
///
/// ```
/// # use std::sync::Arc;
@@ -207,15 +275,17 @@ impl From<&SchemaRef> for TableSchema {
pub struct TableSchemaBuilder {
file_schema: SchemaRef,
table_partition_cols: Fields,
+ virtual_columns: Fields,
}
impl TableSchemaBuilder {
/// Create a builder for a `TableSchema` over the given file schema, with no
- /// partition columns yet.
+ /// partition or virtual columns yet.
pub fn new(file_schema: SchemaRef) -> Self {
Self {
file_schema,
table_partition_cols: Fields::empty(),
+ virtual_columns: Fields::empty(),
}
}
@@ -231,13 +301,39 @@ impl TableSchemaBuilder {
self
}
- /// Build the [`TableSchema`], computing the full `file + partition` schema once.
+ /// Set the virtual columns, replacing any previously set.
+ ///
+ /// Virtual columns are produced by the file reader (e.g. Parquet
+ /// `row_number`) and appended at the end of the table schema. Each field
+ /// must carry an arrow virtual extension type so the reader can recognize
+ /// it.
+ ///
+ /// Accepts anything convertible into [`Fields`] (e.g. `Vec<FieldRef>`).
+ pub fn with_virtual_columns(mut self, virtual_columns: impl Into<Fields>) -> Self {
+ self.virtual_columns = virtual_columns.into();
+ self
+ }
+
+ /// Build the [`TableSchema`], computing the full
+ /// `file + partition + virtual` schema once.
pub fn build(self) -> TableSchema {
+ debug_assert!(
+ self.virtual_columns.iter().enumerate().all(|(i, v)| {
+ let name = v.name();
+ !self.file_schema.fields().iter().any(|f| f.name() == name)
+ && !self.table_partition_cols.iter().any(|p| p.name() == name)
+ && !self.virtual_columns[..i].iter().any(|w| w.name() == name)
+ }),
+ "virtual column name collides with an existing file, partition, or virtual column"
+ );
+
let mut builder = SchemaBuilder::from(self.file_schema.as_ref());
builder.extend(self.table_partition_cols.iter().cloned());
+ builder.extend(self.virtual_columns.iter().cloned());
TableSchema {
file_schema: self.file_schema,
table_partition_cols: self.table_partition_cols,
+ virtual_columns: self.virtual_columns,
table_schema: Arc::new(builder.finish()),
}
}
@@ -393,4 +489,125 @@ mod tests {
assert_eq!(original.table_partition_cols().len(), 1);
assert_eq!(original.table_partition_cols()[0].name(), "country");
}
+
+ #[test]
+ fn test_builder_with_virtual_columns_layout() {
+ let file_schema = Arc::new(Schema::new(vec![
+ Field::new("user_id", DataType::Int64, false),
+ Field::new("amount", DataType::Float64, false),
+ ]));
+
+ let virtual_cols =
+ vec![Arc::new(Field::new("row_number", DataType::Int64, true))];
+
+ let partition_cols = vec![Arc::new(Field::new("date", DataType::Utf8, false))];
+
+ // Apply virtual columns and partition columns in either order on the
+ // builder; the resulting table schema should always be
+ // [file, partition, virtual].
+ let built_virtual_first = TableSchemaBuilder::new(Arc::clone(&file_schema))
+ .with_virtual_columns(virtual_cols.clone())
+ .with_table_partition_cols(partition_cols.clone())
+ .build();
+
+ let built_partition_first = TableSchemaBuilder::new(Arc::clone(&file_schema))
+ .with_table_partition_cols(partition_cols.clone())
+ .with_virtual_columns(virtual_cols.clone())
+ .build();
+
+ let expected = Schema::new(vec![
+ Field::new("user_id", DataType::Int64, false),
+ Field::new("amount", DataType::Float64, false),
+ Field::new("date", DataType::Utf8, false),
+ Field::new("row_number", DataType::Int64, true),
+ ]);
+
+ for ts in [built_virtual_first, built_partition_first] {
+ assert_eq!(ts.table_schema().as_ref(), &expected);
+ assert_eq!(ts.virtual_columns().len(), 1);
+ assert_eq!(ts.virtual_columns()[0].name(), "row_number");
+ assert_eq!(ts.table_partition_cols().len(), 1);
+ assert_eq!(ts.file_schema().fields().len(), 2);
+ }
+ }
+
+ #[test]
+ #[should_panic(expected = "virtual column name collides")]
+ #[cfg(debug_assertions)]
+ fn test_virtual_column_collides_with_file_schema_panics_in_debug() {
+ let file_schema = Arc::new(Schema::new(vec![Field::new(
+ "row_number",
+ DataType::Int64,
+ false,
+ )]));
+ let _ = TableSchemaBuilder::new(file_schema)
+ .with_virtual_columns(vec![Arc::new(Field::new(
+ "row_number",
+ DataType::Int64,
+ true,
+ ))])
+ .build();
+ }
+
+ #[test]
+ #[should_panic(expected = "virtual column name collides")]
+ #[cfg(debug_assertions)]
+ fn test_virtual_column_collides_with_partition_panics_in_debug() {
+ let file_schema = Arc::new(Schema::new(vec![Field::new(
+ "user_id",
+ DataType::Int64,
+ false,
+ )]));
+ let partition_cols =
+ vec![Arc::new(Field::new("row_number", DataType::Utf8, false))];
+ let _ = TableSchemaBuilder::new(file_schema)
+ .with_table_partition_cols(partition_cols)
+ .with_virtual_columns(vec![Arc::new(Field::new(
+ "row_number",
+ DataType::Int64,
+ true,
+ ))])
+ .build();
+ }
+
+ #[test]
+ #[should_panic(expected = "virtual column name collides")]
+ #[cfg(debug_assertions)]
+ fn test_duplicate_virtual_columns_panic_in_debug() {
+ let file_schema = Arc::new(Schema::new(vec![Field::new(
+ "user_id",
+ DataType::Int64,
+ false,
+ )]));
+ let _ = TableSchemaBuilder::new(file_schema)
+ .with_virtual_columns(vec![
+ Arc::new(Field::new("vc", DataType::Int64, true)),
+ Arc::new(Field::new("vc", DataType::Int64, true)),
+ ])
+ .build();
+ }
+
+ #[test]
+ #[should_panic(expected = "virtual column name collides")]
+ #[cfg(debug_assertions)]
+ fn test_partition_column_added_after_colliding_virtual_panics_in_debug() {
+ // Builder order is irrelevant: collision check runs in build().
+ let file_schema = Arc::new(Schema::new(vec![Field::new(
+ "user_id",
+ DataType::Int64,
+ false,
+ )]));
+ let _ = TableSchemaBuilder::new(file_schema)
+ .with_virtual_columns(vec![Arc::new(Field::new(
+ "row_number",
+ DataType::Int64,
+ true,
+ ))])
+ .with_table_partition_cols(vec![Arc::new(Field::new(
+ "row_number",
+ DataType::Utf8,
+ false,
+ ))])
+ .build();
+ }
}