Apache DataFusion SQL Query Engine
chore: Hide internal public utility APIs (#25963)
Yongting You4e6e564e586f
diff --git a/datafusion-examples/.examples-docs-check.E3y1ZS/README.md b/datafusion-examples/.examples-docs-check.E3y1ZS/README.mdnew file mode 100644index 0000000000..e69de29bb2diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex ba813253bc..9d360d052f 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -2854,12 +2854,18 @@ impl Filter {/// Create a new filter operator.////// Skips the type-checking, window function check and dealiasing done in- /// [Self::try_new]. For internal use in DataFusion only.+ /// [Self::try_new].////// **Preconditions:**/// - the `predicate` expression returns a boolean value/// - the `predicate` expression is not aliased/// - the `predicate` expression contains no window function calls+ ///+ /// # Public Only for Internal Use:+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis#[doc(hidden)]pub fn new(predicate: Expr, input: Arc<LogicalPlan>) -> Self {Self { predicate, input }diff --git a/datafusion/physical-expr-common/src/regex.rs b/datafusion/physical-expr-common/src/regex.rsindex e0c32f0473..47761edf41 100644--- a/datafusion/physical-expr-common/src/regex.rs+++ b/datafusion/physical-expr-common/src/regex.rs@@ -106,8 +106,11 @@ pub fn compile_regex(/// compiles a single pattern up front, before it reads any value, takes/// `None`: it compiles that pattern whatever the values are.///-/// This is `pub` only so that the crates that call the kernels can reach it.-// Not public API.+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis#[doc(hidden)]pub fn explain_regexp_kernel_error(function_name: &str,diff --git a/datafusion/physical-expr/src/aggregate.rs b/datafusion/physical-expr/src/aggregate.rsindex 7b72a2622a..71150d330a 100644--- a/datafusion/physical-expr/src/aggregate.rs+++ b/datafusion/physical-expr/src/aggregate.rs@@ -353,6 +353,11 @@ impl AggregateExprBuilder {self}+ /// # Public Only for Internal Use:+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis#[doc(hidden)]pub fn human_display_alias(mut self, alias: impl Into<String>) -> Self {let alias = alias.into();@@ -702,6 +707,11 @@ impl AggregateFunctionExpr {.map(AggregateHumanDisplay::expression)}+ /// # Public Only for Internal Use:+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis#[doc(hidden)]pub fn human_display_alias(&self) -> Option<&str> {self.human_displaydiff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex d4fcccbc9f..4284e7e0f9 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -936,8 +936,9 @@ impl AggregateExec {////// # Public Only for Internal Use:///- /// This is not part of the supported public API, it's made public for internal- /// optimizer usage.+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis////// # TopK Optimization Overview///@@ -1142,6 +1143,13 @@ impl AggregateExec {/// leaves the aggregate in a consistent state. Ineligible aggregates return/// `None`. Eligible aggregates are returned unchanged when the limit is zero/// or no tighter than the existing limit.+ ///+ /// # Public Only for Internal Use:+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+ #[doc(hidden)]pub fn try_optimize_distinct_soft_limit(mut self,limit: usize,@@ -1167,6 +1175,13 @@ impl AggregateExec {////// The caller must preserve the schema, filters, and ordering requirements./// Changing expressions in a TopK aggregate drops its specialization.+ ///+ /// # Public Only for Internal Use:+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+ #[doc(hidden)]pub fn with_new_aggr_exprs(&self,aggr_expr: impl Into<Arc<[Arc<AggregateFunctionExpr>]>>,diff --git a/datafusion/physical-plan/src/distribution_requirements.rs b/datafusion/physical-plan/src/distribution_requirements.rsindex f7a17ddc7b..a8bfddf06a 100644--- a/datafusion/physical-plan/src/distribution_requirements.rs+++ b/datafusion/physical-plan/src/distribution_requirements.rs@@ -188,6 +188,12 @@ impl InputDistributionRequirements {/// Independent per-child requirements are intentionally ignored here, use/// [`Self::child_satisfaction`] for those checks. An empty result means all/// co-partitioning requirements are satisfied.+ ///+ /// # Public Only for Internal Use:+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis#[doc(hidden)]pub fn unsatisfied_co_partitioned_children(&self,diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rsindex 75b05f0b90..51142e9d8e 100644--- a/datafusion/physical-plan/src/joins/cross_join.rs+++ b/datafusion/physical-plan/src/joins/cross_join.rs@@ -187,6 +187,19 @@ impl CrossJoinExec {/// This function should be called BEFORE inserting any repartitioning/// operators on the join's children. Check [`super::HashJoinExec::swap_inputs`]/// for more details.+ ///+ /// # Public Only for Internal Use:+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// Direct use of this API in downstream projects is discouraged because+ /// correctness depends on strict preconditions. See the notes above for+ /// correct usage.+ ///+ /// This API may change frequently as internal join optimizations evolve.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+ #[doc(hidden)]pub fn swap_inputs(&self) -> Result<Arc<dyn ExecutionPlan>> {let new_join =CrossJoinExec::new(Arc::clone(&self.right), Arc::clone(&self.left));diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex f6ff292556..4e404f2853 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -1496,6 +1496,19 @@ impl HashJoinExec {/// physical optimizer rule to determine a good join order, which is/// executed before the `EnforceDistribution` rule (the rule that may/// insert `RepartitionExec` operators).+ ///+ /// # Public Only for Internal Use:+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// Direct use of this API in downstream projects is discouraged because+ /// correctness depends on strict preconditions. See the notes above for+ /// correct usage.+ ///+ /// This API may change frequently as internal join optimizations evolve.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+ #[doc(hidden)]pub fn swap_inputs(&self,partition_mode: PartitionMode,diff --git a/datafusion/physical-plan/src/joins/join_hash_map.rs b/datafusion/physical-plan/src/joins/join_hash_map.rsindex 8129e04397..3f58756fc3 100644--- a/datafusion/physical-plan/src/joins/join_hash_map.rs+++ b/datafusion/physical-plan/src/joins/join_hash_map.rs@@ -97,11 +97,16 @@ use hashbrown::hash_table::Entry::{Occupied, Vacant};/// At runtime we choose between using `JoinHashMapU32` and `JoinHashMapU64` which oth implement/// `JoinHashMapType`.///-/// ## Note on use of this trait as a public API-/// This is currently a public trait but is mainly intended for internal use within DataFusion./// For example, we may compare references to `JoinHashMapType` implementations by pointer equality/// rather than deep equality of contents, as deep equality would be expensive and in our usage/// patterns it is impossible for two different hash maps to have identical contents in a practical sense.+///+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]pub trait JoinHashMapType: Send + Sync {fn extend_zero(&mut self, len: usize);diff --git a/datafusion/physical-plan/src/joins/mod.rs b/datafusion/physical-plan/src/joins/mod.rsindex c5d5fbe543..7bfd98ebb2 100644--- a/datafusion/physical-plan/src/joins/mod.rs+++ b/datafusion/physical-plan/src/joins/mod.rs@@ -79,6 +79,13 @@ use utils::JoinHashMapType;/// contains rows.////// [`NullEquality::NullEqualsNothing`]: datafusion_common::NullEquality::NullEqualsNothing+///+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]pub enum Map {HashMap(Box<dyn JoinHashMapType>),ArrayMap(ArrayMap),diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rsindex 0b3e059302..dfa7eb0d8f 100644--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs@@ -478,6 +478,19 @@ impl NestedLoopJoinExec {/// This function should be called BEFORE inserting any repartitioning/// operators on the join's children. Check [`super::HashJoinExec::swap_inputs`]/// for more details.+ ///+ /// # Public Only for Internal Use:+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// Direct use of this API in downstream projects is discouraged because+ /// correctness depends on strict preconditions. See the notes above for+ /// correct usage.+ ///+ /// This API may change frequently as internal join optimizations evolve.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+ #[doc(hidden)]pub fn swap_inputs(&self) -> Result<Arc<dyn ExecutionPlan>> {let left = self.left();let right = self.right();diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rsindex c0e57a93a4..64f1817184 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs@@ -349,6 +349,19 @@ impl SortMergeJoinExec {/// This function should be called BEFORE inserting any repartitioning/// operators on the join's children. Check [`super::super::HashJoinExec::swap_inputs`]/// for more details.+ ///+ /// # Public Only for Internal Use:+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// Direct use of this API in downstream projects is discouraged because+ /// correctness depends on strict preconditions. See the notes above for+ /// correct usage.+ ///+ /// This API may change frequently as internal join optimizations evolve.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+ #[doc(hidden)]pub fn swap_inputs(&self) -> Result<Arc<dyn ExecutionPlan>> {let left = self.left();let right = self.right();diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rsindex d7398c2a90..a2c4715d2c 100644--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs@@ -336,6 +336,13 @@ impl SymmetricHashJoinExec {}/// Check if order information covers every column in the filter expression.+ ///+ /// # Public Only for Internal Use:+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+ #[doc(hidden)]pub fn check_if_order_information_available(&self) -> Result<bool> {if let Some(filter) = self.filter() {let left = self.left();diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rsindex a56aab6d25..2dfc8cd055 100644--- a/datafusion/physical-plan/src/joins/utils.rs+++ b/datafusion/physical-plan/src/joins/utils.rs@@ -37,6 +37,13 @@ use crate::{};// compatibilitypub use super::join_filter::JoinFilter;+///+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]pub use super::join_hash_map::JoinHashMapType;pub use crate::joins::{JoinOn, JoinOnRef};diff --git a/datafusion/physical-plan/src/proto.rs b/datafusion/physical-plan/src/proto.rsindex 7640d76c3e..a3f8182198 100644--- a/datafusion/physical-plan/src/proto.rs+++ b/datafusion/physical-plan/src/proto.rs@@ -81,9 +81,11 @@ use crate::ExecutionPlan;/// Implemented by `datafusion-proto`. Plan authors never name this trait; they/// call methods on [`ExecutionPlanEncodeCtx`] instead.///-/// **Not public API.** `pub` only because the implementors live in another-/// crate; `#[doc(hidden)]` records that, so encoding primitives can be added-/// here as the serialization hooks grow without breaking downstream code.+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis#[doc(hidden)]pub trait ExecutionPlanEncode {/// Serialize a child execution plan (recursing through the central@@ -111,9 +113,11 @@ pub trait ExecutionPlanEncode {/// Implemented by `datafusion-proto`. Plan authors never name this trait; they/// call methods on [`ExecutionPlanDecodeCtx`] instead.///-/// **Not public API.** `pub` only because the implementors live in another-/// crate; `#[doc(hidden)]` records that, so decoding primitives can be added-/// here as the serialization hooks grow without breaking downstream code.+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis#[doc(hidden)]pub trait ExecutionPlanDecode {/// Deserialize a child execution plan (recursing through the centraldiff --git a/docs/source/contributor-guide/api-health.md b/docs/source/contributor-guide/api-health.mdindex e74d74ea51..d5452dd09e 100644--- a/docs/source/contributor-guide/api-health.md+++ b/docs/source/contributor-guide/api-health.md@@ -63,18 +63,18 @@ Do not expose internal APIs solely for tests or microbenchmarks. Some legacy coddoes so, but new tests and benchmarks should exercise observable behavior tosimplify maintenance.-For APIs intended only for internal use, add `#[doc(hidden)]` and a doc comment-section headed `# Public Only for Internal Use:`. Name the crate or component-that requires access and explain why the API is not intended for downstream use.-For example:+For APIs intended only for internal use, add `#[doc(hidden)]` and use the+following doc comment format, including the API policy link:```txtimpl HashTableLookupExpr {/// ...////// # Public Only for Internal Use:- /// `datafusion-proto` tests require this constructor, but it is not part of- /// the supported public API.+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis#[doc(hidden)]pub fn new(...) {...}}