Apache DataFusion SQL Query Engine
Doc: Add more blog links to doc comments (#19837)
Andrew Lamb0c959d6d919b
diff --git a/datafusion/common/src/metadata.rs b/datafusion/common/src/metadata.rsindex eb687bde07..d6d8fb7b0e 100644--- a/datafusion/common/src/metadata.rs+++ b/datafusion/common/src/metadata.rs@@ -171,6 +171,10 @@ pub fn format_type_and_metadata(/// // Add any metadata from `FieldMetadata` to `Field`/// let updated_field = metadata.add_to_field(field);/// ```+///+/// For more background, please also see the [Implementing User Defined Types and Custom Metadata in DataFusion blog]+///+/// [Implementing User Defined Types and Custom Metadata in DataFusion blog]: https://datafusion.apache.org/blog/2025/09/21/custom-types-using-metadata#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]pub struct FieldMetadata {/// The inner metadata of a literal expression, which is a map of stringdiff --git a/datafusion/datasource-parquet/src/access_plan.rs b/datafusion/datasource-parquet/src/access_plan.rsindex 570792d40e..44911fcf2a 100644--- a/datafusion/datasource-parquet/src/access_plan.rs+++ b/datafusion/datasource-parquet/src/access_plan.rs@@ -82,6 +82,10 @@ use parquet::file::metadata::RowGroupMetaData;/// └───────────────────┘/// Row Group 3/// ```+///+/// For more background, please also see the [Embedding User-Defined Indexes in Apache Parquet Files blog]+///+/// [Embedding User-Defined Indexes in Apache Parquet Files blog]: https://datafusion.apache.org/blog/2025/07/14/user-defined-parquet-indexes#[derive(Debug, Clone, PartialEq)]pub struct ParquetAccessPlan {/// How to access the i-th row groupdiff --git a/datafusion/expr-common/src/groups_accumulator.rs b/datafusion/expr-common/src/groups_accumulator.rsindex 860e69245a..08c9f01f13 100644--- a/datafusion/expr-common/src/groups_accumulator.rs+++ b/datafusion/expr-common/src/groups_accumulator.rs@@ -89,6 +89,9 @@ impl EmitTo {/// optional and is harder to implement than `Accumulator`, but can be much/// faster for queries with many group values. See the [Aggregating Millions of/// Groups Fast blog] for more background.+/// For more background, please also see the [Aggregating Millions of Groups Fast in Apache Arrow DataFusion 28.0.0 blog]+///+/// [Aggregating Millions of Groups Fast in Apache Arrow DataFusion 28.0.0 blog]: https://datafusion.apache.org/blog/2023/08/05/datafusion_fast_grouping////// [`NullState`] can help keep the state for groups that have not seen any/// values and produce the correct output for those groups.diff --git a/datafusion/expr/src/partition_evaluator.rs b/datafusion/expr/src/partition_evaluator.rsindex 3e7ba5d4f5..0671f31f6d 100644--- a/datafusion/expr/src/partition_evaluator.rs+++ b/datafusion/expr/src/partition_evaluator.rs@@ -86,6 +86,10 @@ use crate::window_state::WindowAggState;/// [`uses_window_frame`]: Self::uses_window_frame/// [`include_rank`]: Self::include_rank/// [`supports_bounded_execution`]: Self::supports_bounded_execution+///+/// For more background, please also see the [User defined Window Functions in DataFusion blog]+///+/// [User defined Window Functions in DataFusion blog]: https://datafusion.apache.org/blog/2025/04/19/user-defined-window-functionspub trait PartitionEvaluator: Debug + Send {/// When the window frame has a fixed beginning (e.g UNBOUNDED/// PRECEDING), some functions such as FIRST_VALUE, LAST_VALUE anddiff --git a/datafusion/expr/src/planner.rs b/datafusion/expr/src/planner.rsindex 042c48ffff..837a9eefe2 100644--- a/datafusion/expr/src/planner.rs+++ b/datafusion/expr/src/planner.rs@@ -139,6 +139,10 @@ pub trait ContextProvider {}/// Customize planning of SQL AST expressions to [`Expr`]s+///+/// For more background, please also see the [Extending SQL in DataFusion: from ->> to TABLESAMPLE blog]+///+/// [Extending SQL in DataFusion: from ->> to TABLESAMPLE blog]: https://datafusion.apache.org/blog/2026/01/12/extending-sqlpub trait ExprPlanner: Debug + Send + Sync {/// Plan the binary operation between two expressions, returns original/// BinaryExpr if not possible@@ -369,6 +373,9 @@ pub enum RelationPlanning {/// Customize planning SQL table factors to [`LogicalPlan`]s.#[cfg(feature = "sql")]+/// For more background, please also see the [Extending SQL in DataFusion: from ->> to TABLESAMPLE blog]+///+/// [Extending SQL in DataFusion: from ->> to TABLESAMPLE blog]: https://datafusion.apache.org/blog/2026/01/12/extending-sqlpub trait RelationPlanner: Debug + Send + Sync {/// Plan a table factor into a [`LogicalPlan`].///@@ -420,6 +427,9 @@ pub trait RelationPlannerContext {/// Customize planning SQL types to DataFusion (Arrow) types.#[cfg(feature = "sql")]+/// For more background, please also see the [Extending SQL in DataFusion: from ->> to TABLESAMPLE blog]+///+/// [Extending SQL in DataFusion: from ->> to TABLESAMPLE blog]: https://datafusion.apache.org/blog/2026/01/12/extending-sqlpub trait TypePlanner: Debug + Send + Sync {/// Plan SQL [`sqlparser::ast::DataType`] to DataFusion [`DataType`]///diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters.rs b/datafusion/physical-expr/src/expressions/dynamic_filters.rsindex 643745ac0f..6c961e3bb0 100644--- a/datafusion/physical-expr/src/expressions/dynamic_filters.rs+++ b/datafusion/physical-expr/src/expressions/dynamic_filters.rs@@ -51,6 +51,10 @@ impl FilterState {/// Any `ExecutionPlan` that uses this expression and holds a reference to it internally should probably also/// implement `ExecutionPlan::reset_state` to remain compatible with recursive queries and other situations where/// the same `ExecutionPlan` is reused with different data.+///+/// For more background, please also see the [Dynamic Filters: Passing Information Between Operators During Execution for 25x Faster Queries blog]+///+/// [Dynamic Filters: Passing Information Between Operators During Execution for 25x Faster Queries blog]: https://datafusion.apache.org/blog/2025/09/10/dynamic-filters#[derive(Debug)]pub struct DynamicFilterPhysicalExpr {/// The original children of this PhysicalExpr, if any.diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rsindex a1fad86777..ce54a451ac 100644--- a/datafusion/physical-plan/src/coop.rs+++ b/datafusion/physical-plan/src/coop.rs@@ -22,10 +22,15 @@//! A single call to `poll_next` on a top-level [`Stream`] may potentially perform a lot of work//! before it returns a `Poll::Pending`. Think for instance of calculating an aggregation over a//! large dataset.+//!//! If a `Stream` runs for a long period of time without yielding back to the Tokio executor,//! it can starve other tasks waiting on that executor to execute them.//! Additionally, this prevents the query execution from being cancelled.//!+//! For more background, please also see the [Using Rust async for Query Execution and Cancelling Long-Running Queries blog]+//!+//! [Using Rust async for Query Execution and Cancelling Long-Running Queries blog]: https://datafusion.apache.org/blog/2025/06/30/cancellation+//!//! To ensure that `Stream` implementations yield regularly, operators can insert explicit yield//! points using the utilities in this module. For most operators this is **not** necessary. The//! `Stream`s of the built-in DataFusion operators that generate (rather than manipulate)diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex fb56170c2b..612c7bb27d 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -746,6 +746,10 @@ impl BatchPartitioner {/// system Paper](https://dl.acm.org/doi/pdf/10.1145/93605.98720)/// which uses the term "Exchange" for the concept of repartitioning/// data across threads.+///+/// For more background, please also see the [Optimizing Repartitions in DataFusion] blog.+///+/// [Optimizing Repartitions in DataFusion]: https://datafusion.apache.org/blog/2025/12/15/avoid-consecutive-repartitions#[derive(Debug, Clone)]pub struct RepartitionExec {/// Input execution plandiff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rsindex ebac497f4f..4b93e6a188 100644--- a/datafusion/physical-plan/src/topk/mod.rs+++ b/datafusion/physical-plan/src/topk/mod.rs@@ -131,6 +131,9 @@ pub struct TopK {pub(crate) finished: bool,}+/// For more background, please also see the [Dynamic Filters: Passing Information Between Operators During Execution for 25x Faster Queries blog]+///+/// [Dynamic Filters: Passing Information Between Operators During Execution for 25x Faster Queries blog]: https://datafusion.apache.org/blog/2025/09/10/dynamic-filters#[derive(Debug, Clone)]pub struct TopKDynamicFilters {/// The current *global* threshold for the dynamic filter.