Skip to content

Apache DataFusion SQL Query Engine

More documentation on `FileSource::table_schema` and `FileSource::projection` (#20242)

Andrew Lamb2e1fc5ccb470
diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rs
index b41a456f1f..a0f82ff7a9 100644
--- a/datafusion/datasource/src/file.rs
+++ b/datafusion/datasource/src/file.rs
@@ -39,16 +39,17 @@ use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
use object_store::ObjectStore;
-/// Helper function to convert any type implementing FileSource to Arc<dyn FileSource>
+/// Helper function to convert any type implementing [`FileSource`] to `Arc<dyn FileSource>`
pub fn as_file_source<T: FileSource + 'static>(source: T) -> Arc<dyn FileSource> {
Arc::new(source)
}
-/// file format specific behaviors for elements in [`DataSource`]
+/// File format specific behaviors for [`DataSource`]
///
/// # Schema information
/// There are two important schemas for a [`FileSource`]:
-/// 1. [`Self::table_schema`] -- the schema for the overall "table"
+/// 1. [`Self::table_schema`] -- the schema for the overall table
+/// (file data plus partition columns)
/// 2. The logical output schema, comprised of [`Self::table_schema`] with
/// [`Self::projection`] applied
///
@@ -71,13 +72,16 @@ pub trait FileSource: Send + Sync {
/// Any
fn as_any(&self) -> &dyn Any;
- /// Returns the table schema for this file source.
+ /// Returns the table schema for the overall table (including partition columns, if any)
///
- /// This always returns the unprojected schema (the full schema of the data)
+ /// This method returns the unprojected schema: the full schema of the data
/// without [`Self::projection`] applied.
///
/// The output schema of this `FileSource` is this TableSchema
/// with [`Self::projection`] applied.
+ ///
+ /// Use [`ProjectionExprs::project_schema`] to get the projected schema
+ /// after applying the projection.
fn table_schema(&self) -> &crate::table_schema::TableSchema;
/// Initialize new type with batch size configuration
@@ -92,6 +96,9 @@ pub trait FileSource: Send + Sync {
/// Return the projection that will be applied to the output stream on top
/// of [`Self::table_schema`].
+ ///
+ /// Note you can use [`ProjectionExprs::project_schema`] on the table
+ /// schema to get the effective output schema of this source.
fn projection(&self) -> Option<&ProjectionExprs> {
None
}
diff --git a/datafusion/datasource/src/table_schema.rs b/datafusion/datasource/src/table_schema.rs
index a45cdbaaea..5b7fc4727d 100644
--- a/datafusion/datasource/src/table_schema.rs
+++ b/datafusion/datasource/src/table_schema.rs
@@ -20,13 +20,13 @@
use arrow::datatypes::{FieldRef, SchemaBuilder, SchemaRef};
use std::sync::Arc;
-/// Helper to hold table schema information for partitioned data sources.
+/// The overall schema for potentially partitioned data sources.
///
-/// When reading partitioned data (such as Hive-style partitioning), a table's schema
+/// When reading partitioned data (such as Hive-style partitioning), a [`TableSchema`]
/// consists of two parts:
/// 1. **File schema**: The schema of the actual data files on disk
-/// 2. **Partition columns**: Columns that are encoded in the directory structure,
-/// not stored in the files themselves
+/// 2. **Partition columns**: Columns whose values are encoded in the directory structure,
+/// but not stored in the files themselves
///
/// # Example: Partitioned Table
///