Skip to content

Apache DataFusion SQL Query Engine

chore: Hide internal public utility APIs (#25963)

Yongting You4e6e564e586f
diff --git a/datafusion-examples/.examples-docs-check.E3y1ZS/README.md b/datafusion-examples/.examples-docs-check.E3y1ZS/README.md
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs
index ba813253bc..9d360d052f 100644
--- a/datafusion/expr/src/logical_plan/plan.rs
+++ b/datafusion/expr/src/logical_plan/plan.rs
@@ -2854,12 +2854,18 @@ impl Filter {
/// Create a new filter operator.
///
/// Skips the type-checking, window function check and dealiasing done in
- /// [Self::try_new]. For internal use in DataFusion only.
+ /// [Self::try_new].
///
/// **Preconditions:**
/// - the `predicate` expression returns a boolean value
/// - the `predicate` expression is not aliased
/// - the `predicate` expression contains no window function calls
+ ///
+ /// # Public Only for Internal Use:
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
#[doc(hidden)]
pub fn new(predicate: Expr, input: Arc<LogicalPlan>) -> Self {
Self { predicate, input }
diff --git a/datafusion/physical-expr-common/src/regex.rs b/datafusion/physical-expr-common/src/regex.rs
index e0c32f0473..47761edf41 100644
--- a/datafusion/physical-expr-common/src/regex.rs
+++ b/datafusion/physical-expr-common/src/regex.rs
@@ -106,8 +106,11 @@ pub fn compile_regex(
/// compiles a single pattern up front, before it reads any value, takes
/// `None`: it compiles that pattern whatever the values are.
///
-/// This is `pub` only so that the crates that call the kernels can reach it.
-// Not public API.
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
#[doc(hidden)]
pub fn explain_regexp_kernel_error(
function_name: &str,
diff --git a/datafusion/physical-expr/src/aggregate.rs b/datafusion/physical-expr/src/aggregate.rs
index 7b72a2622a..71150d330a 100644
--- a/datafusion/physical-expr/src/aggregate.rs
+++ b/datafusion/physical-expr/src/aggregate.rs
@@ -353,6 +353,11 @@ impl AggregateExprBuilder {
self
}
+ /// # Public Only for Internal Use:
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
#[doc(hidden)]
pub fn human_display_alias(mut self, alias: impl Into<String>) -> Self {
let alias = alias.into();
@@ -702,6 +707,11 @@ impl AggregateFunctionExpr {
.map(AggregateHumanDisplay::expression)
}
+ /// # Public Only for Internal Use:
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
#[doc(hidden)]
pub fn human_display_alias(&self) -> Option<&str> {
self.human_display
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index d4fcccbc9f..4284e7e0f9 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -936,8 +936,9 @@ impl AggregateExec {
///
/// # Public Only for Internal Use:
///
- /// This is not part of the supported public API, it's made public for internal
- /// optimizer usage.
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
///
/// # TopK Optimization Overview
///
@@ -1142,6 +1143,13 @@ impl AggregateExec {
/// leaves the aggregate in a consistent state. Ineligible aggregates return
/// `None`. Eligible aggregates are returned unchanged when the limit is zero
/// or no tighter than the existing limit.
+ ///
+ /// # Public Only for Internal Use:
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+ #[doc(hidden)]
pub fn try_optimize_distinct_soft_limit(
mut self,
limit: usize,
@@ -1167,6 +1175,13 @@ impl AggregateExec {
///
/// The caller must preserve the schema, filters, and ordering requirements.
/// Changing expressions in a TopK aggregate drops its specialization.
+ ///
+ /// # Public Only for Internal Use:
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+ #[doc(hidden)]
pub fn with_new_aggr_exprs(
&self,
aggr_expr: impl Into<Arc<[Arc<AggregateFunctionExpr>]>>,
diff --git a/datafusion/physical-plan/src/distribution_requirements.rs b/datafusion/physical-plan/src/distribution_requirements.rs
index f7a17ddc7b..a8bfddf06a 100644
--- a/datafusion/physical-plan/src/distribution_requirements.rs
+++ b/datafusion/physical-plan/src/distribution_requirements.rs
@@ -188,6 +188,12 @@ impl InputDistributionRequirements {
/// Independent per-child requirements are intentionally ignored here, use
/// [`Self::child_satisfaction`] for those checks. An empty result means all
/// co-partitioning requirements are satisfied.
+ ///
+ /// # Public Only for Internal Use:
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
#[doc(hidden)]
pub fn unsatisfied_co_partitioned_children(
&self,
diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs
index 75b05f0b90..51142e9d8e 100644
--- a/datafusion/physical-plan/src/joins/cross_join.rs
+++ b/datafusion/physical-plan/src/joins/cross_join.rs
@@ -187,6 +187,19 @@ impl CrossJoinExec {
/// This function should be called BEFORE inserting any repartitioning
/// operators on the join's children. Check [`super::HashJoinExec::swap_inputs`]
/// for more details.
+ ///
+ /// # Public Only for Internal Use:
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// Direct use of this API in downstream projects is discouraged because
+ /// correctness depends on strict preconditions. See the notes above for
+ /// correct usage.
+ ///
+ /// This API may change frequently as internal join optimizations evolve.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+ #[doc(hidden)]
pub fn swap_inputs(&self) -> Result<Arc<dyn ExecutionPlan>> {
let new_join =
CrossJoinExec::new(Arc::clone(&self.right), Arc::clone(&self.left));
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index f6ff292556..4e404f2853 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -1496,6 +1496,19 @@ impl HashJoinExec {
/// physical optimizer rule to determine a good join order, which is
/// executed before the `EnforceDistribution` rule (the rule that may
/// insert `RepartitionExec` operators).
+ ///
+ /// # Public Only for Internal Use:
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// Direct use of this API in downstream projects is discouraged because
+ /// correctness depends on strict preconditions. See the notes above for
+ /// correct usage.
+ ///
+ /// This API may change frequently as internal join optimizations evolve.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+ #[doc(hidden)]
pub fn swap_inputs(
&self,
partition_mode: PartitionMode,
diff --git a/datafusion/physical-plan/src/joins/join_hash_map.rs b/datafusion/physical-plan/src/joins/join_hash_map.rs
index 8129e04397..3f58756fc3 100644
--- a/datafusion/physical-plan/src/joins/join_hash_map.rs
+++ b/datafusion/physical-plan/src/joins/join_hash_map.rs
@@ -97,11 +97,16 @@ use hashbrown::hash_table::Entry::{Occupied, Vacant};
/// At runtime we choose between using `JoinHashMapU32` and `JoinHashMapU64` which oth implement
/// `JoinHashMapType`.
///
-/// ## Note on use of this trait as a public API
-/// This is currently a public trait but is mainly intended for internal use within DataFusion.
/// For example, we may compare references to `JoinHashMapType` implementations by pointer equality
/// rather than deep equality of contents, as deep equality would be expensive and in our usage
/// patterns it is impossible for two different hash maps to have identical contents in a practical sense.
+///
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
pub trait JoinHashMapType: Send + Sync {
fn extend_zero(&mut self, len: usize);
diff --git a/datafusion/physical-plan/src/joins/mod.rs b/datafusion/physical-plan/src/joins/mod.rs
index c5d5fbe543..7bfd98ebb2 100644
--- a/datafusion/physical-plan/src/joins/mod.rs
+++ b/datafusion/physical-plan/src/joins/mod.rs
@@ -79,6 +79,13 @@ use utils::JoinHashMapType;
/// contains rows.
///
/// [`NullEquality::NullEqualsNothing`]: datafusion_common::NullEquality::NullEqualsNothing
+///
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
pub enum Map {
HashMap(Box<dyn JoinHashMapType>),
ArrayMap(ArrayMap),
diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs
index 0b3e059302..dfa7eb0d8f 100644
--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs
+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs
@@ -478,6 +478,19 @@ impl NestedLoopJoinExec {
/// This function should be called BEFORE inserting any repartitioning
/// operators on the join's children. Check [`super::HashJoinExec::swap_inputs`]
/// for more details.
+ ///
+ /// # Public Only for Internal Use:
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// Direct use of this API in downstream projects is discouraged because
+ /// correctness depends on strict preconditions. See the notes above for
+ /// correct usage.
+ ///
+ /// This API may change frequently as internal join optimizations evolve.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+ #[doc(hidden)]
pub fn swap_inputs(&self) -> Result<Arc<dyn ExecutionPlan>> {
let left = self.left();
let right = self.right();
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
index c0e57a93a4..64f1817184 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
@@ -349,6 +349,19 @@ impl SortMergeJoinExec {
/// This function should be called BEFORE inserting any repartitioning
/// operators on the join's children. Check [`super::super::HashJoinExec::swap_inputs`]
/// for more details.
+ ///
+ /// # Public Only for Internal Use:
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// Direct use of this API in downstream projects is discouraged because
+ /// correctness depends on strict preconditions. See the notes above for
+ /// correct usage.
+ ///
+ /// This API may change frequently as internal join optimizations evolve.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+ #[doc(hidden)]
pub fn swap_inputs(&self) -> Result<Arc<dyn ExecutionPlan>> {
let left = self.left();
let right = self.right();
diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
index d7398c2a90..a2c4715d2c 100644
--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
@@ -336,6 +336,13 @@ impl SymmetricHashJoinExec {
}
/// Check if order information covers every column in the filter expression.
+ ///
+ /// # Public Only for Internal Use:
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+ #[doc(hidden)]
pub fn check_if_order_information_available(&self) -> Result<bool> {
if let Some(filter) = self.filter() {
let left = self.left();
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs
index a56aab6d25..2dfc8cd055 100644
--- a/datafusion/physical-plan/src/joins/utils.rs
+++ b/datafusion/physical-plan/src/joins/utils.rs
@@ -37,6 +37,13 @@ use crate::{
};
// compatibility
pub use super::join_filter::JoinFilter;
+///
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
pub use super::join_hash_map::JoinHashMapType;
pub use crate::joins::{JoinOn, JoinOnRef};
diff --git a/datafusion/physical-plan/src/proto.rs b/datafusion/physical-plan/src/proto.rs
index 7640d76c3e..a3f8182198 100644
--- a/datafusion/physical-plan/src/proto.rs
+++ b/datafusion/physical-plan/src/proto.rs
@@ -81,9 +81,11 @@ use crate::ExecutionPlan;
/// Implemented by `datafusion-proto`. Plan authors never name this trait; they
/// call methods on [`ExecutionPlanEncodeCtx`] instead.
///
-/// **Not public API.** `pub` only because the implementors live in another
-/// crate; `#[doc(hidden)]` records that, so encoding primitives can be added
-/// here as the serialization hooks grow without breaking downstream code.
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
#[doc(hidden)]
pub trait ExecutionPlanEncode {
/// Serialize a child execution plan (recursing through the central
@@ -111,9 +113,11 @@ pub trait ExecutionPlanEncode {
/// Implemented by `datafusion-proto`. Plan authors never name this trait; they
/// call methods on [`ExecutionPlanDecodeCtx`] instead.
///
-/// **Not public API.** `pub` only because the implementors live in another
-/// crate; `#[doc(hidden)]` records that, so decoding primitives can be added
-/// here as the serialization hooks grow without breaking downstream code.
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
#[doc(hidden)]
pub trait ExecutionPlanDecode {
/// Deserialize a child execution plan (recursing through the central
diff --git a/docs/source/contributor-guide/api-health.md b/docs/source/contributor-guide/api-health.md
index e74d74ea51..d5452dd09e 100644
--- a/docs/source/contributor-guide/api-health.md
+++ b/docs/source/contributor-guide/api-health.md
@@ -63,18 +63,18 @@ Do not expose internal APIs solely for tests or microbenchmarks. Some legacy cod
does so, but new tests and benchmarks should exercise observable behavior to
simplify maintenance.
-For APIs intended only for internal use, add `#[doc(hidden)]` and a doc comment
-section headed `# Public Only for Internal Use:`. Name the crate or component
-that requires access and explain why the API is not intended for downstream use.
-For example:
+For APIs intended only for internal use, add `#[doc(hidden)]` and use the
+following doc comment format, including the API policy link:
```txt
impl HashTableLookupExpr {
/// ...
///
/// # Public Only for Internal Use:
- /// `datafusion-proto` tests require this constructor, but it is not part of
- /// the supported public API.
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
#[doc(hidden)]
pub fn new(...) {...}
}