Apache DataFusion SQL Query Engine
chore: Hide test-only internal public APIs (#25695)
Yongting You2b1fcae86640
diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rsindex 9cfa32e0d0..6fbff7dd5d 100644--- a/datafusion/catalog/src/memory/table.rs+++ b/datafusion/catalog/src/memory/table.rs@@ -71,7 +71,12 @@ pub use datafusion_datasource::memory::PartitionData;#[derive(Debug)]pub struct MemTable {schema: SchemaRef,- // batches used to be pub(crate), but it's needed to be public for the tests+ /// # Public Only for Internal Use:+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+ #[doc(hidden)]pub batches: Vec<PartitionData>,constraints: Constraints,column_defaults: HashMap<String, Expr>,diff --git a/datafusion/core/src/lib.rs b/datafusion/core/src/lib.rsindex 28ea4b4490..ff831a2676 100644--- a/datafusion/core/src/lib.rs+++ b/datafusion/core/src/lib.rs@@ -855,7 +855,19 @@ pub mod physical_plan {}// Reexport testing macros for compatibility+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]pub use datafusion_common::assert_batches_eq;+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]pub use datafusion_common::assert_batches_sorted_eq;/// re-export of [`datafusion_sql`] crate@@ -895,10 +907,22 @@ pub mod variable {pub use datafusion_expr::var_provider::{VarProvider, VarType};}+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis#[cfg(not(target_arch = "wasm32"))]+#[doc(hidden)]pub mod test;mod schema_equivalence;+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]pub mod test_util;#[cfg(doctest)]diff --git a/datafusion/datasource/src/mod.rs b/datafusion/datasource/src/mod.rsindex d0bf64d81c..6135835616 100644--- a/datafusion/datasource/src/mod.rs+++ b/datafusion/datasource/src/mod.rs@@ -413,8 +413,6 @@ impl From<ObjectMeta> for PartitionedFile {/// Generates test files with min-max statistics in different overlap patterns.///-/// Used by tests and benchmarks.-////// # Overlap Factors////// The `overlap_factor` parameter controls how much the value ranges in generated test files overlap:@@ -450,6 +448,13 @@ impl From<ObjectMeta> for PartitionedFile {/// File 2: [40, 140]/// File 3: [60, 160]/// File 4: [80, 180]+///+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec<FileGroup> {let mut files = Vec::with_capacity(num_files);if num_files == 0 {@@ -500,8 +505,14 @@ pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec<FileGrovec![FileGroup::new(files)]}-// Helper function to verify that files within each group maintain sort order-/// Used by tests and benchmarks+/// Verifies that files within each group maintain sort order.+///+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]pub fn verify_sort_integrity(file_groups: &[FileGroup]) -> bool {for group in file_groups {// Known-empty files contribute no rows and may not have min/maxdiff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rsindex ed9b4df4ce..48ed38e8f4 100644--- a/datafusion/expr/src/lib.rs+++ b/datafusion/expr/src/lib.rs@@ -88,6 +88,12 @@ pub mod preimage;pub mod ptr_eq;#[cfg(not(feature = "sql"))]pub mod sql;+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]pub mod test;pub mod tree_node;pub mod type_coercion;diff --git a/datafusion/functions-aggregate-common/src/tdigest.rs b/datafusion/functions-aggregate-common/src/tdigest.rsindex 860f56f6c0..ec9b988df2 100644--- a/datafusion/functions-aggregate-common/src/tdigest.rs+++ b/datafusion/functions-aggregate-common/src/tdigest.rs@@ -206,7 +206,12 @@ impl TDigest {v.clamp(min, max)}- // public for testing in other modules+ /// # Public Only for Internal Use:+ ///+ /// This is not a public API and is for internal use only; see [API policy] for details.+ ///+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+ #[doc(hidden)]pub fn merge_unsorted_f64(&self, unsorted_values: Vec<f64>) -> TDigest {let mut values = unsorted_values;values.sort_by(|a, b| a.total_cmp(b));diff --git a/datafusion/physical-expr/src/intervals/mod.rs b/datafusion/physical-expr/src/intervals/mod.rsindex 9752ca27b5..86f8d2b463 100644--- a/datafusion/physical-expr/src/intervals/mod.rs+++ b/datafusion/physical-expr/src/intervals/mod.rs@@ -18,5 +18,11 @@//! Interval arithmetic and constraint propagation librarypub mod cp_solver;+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]pub mod test_utils;pub mod utils;diff --git a/datafusion/physical-plan/src/joins/hash_join/mod.rs b/datafusion/physical-plan/src/joins/hash_join/mod.rsindex 7b50435e18..aef27fa305 100644--- a/datafusion/physical-plan/src/joins/hash_join/mod.rs+++ b/datafusion/physical-plan/src/joins/hash_join/mod.rs@@ -18,7 +18,14 @@//! [`HashJoinExec`] Partitioned Hash Join Operatorpub use exec::{HashJoinExec, HashJoinExecBuilder, PreparedHashJoinBuild};-pub use partitioned_hash_eval::{HashExpr, HashTableLookupExpr, SeededRandomState};+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]+pub use partitioned_hash_eval::HashTableLookupExpr;+pub use partitioned_hash_eval::{HashExpr, SeededRandomState};mod exec;mod inlist_builder;diff --git a/datafusion/physical-plan/src/joins/mod.rs b/datafusion/physical-plan/src/joins/mod.rsindex 0d2480c131..8ffc6a9467 100644--- a/datafusion/physical-plan/src/joins/mod.rs+++ b/datafusion/physical-plan/src/joins/mod.rs@@ -24,9 +24,15 @@ use arrow::array::BooleanBufferBuilder;pub use asof_join::{AsOfJoinExec, AsOfMatchExpr};pub use cross_join::CrossJoinExec;use datafusion_physical_expr::PhysicalExprRef;+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]+pub use hash_join::HashTableLookupExpr;pub use hash_join::{- HashExpr, HashJoinExec, HashJoinExecBuilder, HashTableLookupExpr,- PreparedHashJoinBuild, SeededRandomState,+ HashExpr, HashJoinExec, HashJoinExecBuilder, PreparedHashJoinBuild, SeededRandomState,};pub use nested_loop_join::{NestedLoopJoinExec, NestedLoopJoinExecBuilder};use parking_lot::Mutex;@@ -52,8 +58,12 @@ mod array_map;mod join_filter;/// Hash map implementations for join operations.///-/// Note: This module is public for internal testing purposes only-/// and is not guaranteed to be stable across versions.+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]pub mod join_hash_map;use array_map::ArrayMap;diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rsindex 5ff6cec374..40489eada3 100644--- a/datafusion/physical-plan/src/lib.rs+++ b/datafusion/physical-plan/src/lib.rs@@ -115,4 +115,10 @@ pub mod udaf {pub use datafusion_physical_expr::aggregate::AggregateFunctionExpr;}+/// # Public Only for Internal Use:+///+/// This is not a public API and is for internal use only; see [API policy] for details.+///+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis+#[doc(hidden)]pub mod test;diff --git a/docs/source/contributor-guide/api-health.md b/docs/source/contributor-guide/api-health.mdindex 8737df280f..e74d74ea51 100644--- a/docs/source/contributor-guide/api-health.md+++ b/docs/source/contributor-guide/api-health.md@@ -43,6 +43,10 @@ Examples of non-breaking changes include:- Marking a function as deprecated (`#[deprecated]`)- Adding a new function to a `trait` with a default implementation+<!-- Keep this anchor stable: it is linked from Rust API documentation. -->++<a id="internal-public-apis"></a>+### DataFusion-internal Public APIsSome internal implementation details require `pub` visibility because they are@@ -55,6 +59,10 @@ notice or a deprecation period. Examples include:2. Operator APIs required by the optimizer to inspect or rewrite execution plansacross crate boundaries.+Do not expose internal APIs solely for tests or microbenchmarks. Some legacy code+does so, but new tests and benchmarks should exercise observable behavior to+simplify maintenance.+For APIs intended only for internal use, add `#[doc(hidden)]` and a doc commentsection headed `# Public Only for Internal Use:`. Name the crate or componentthat requires access and explain why the API is not intended for downstream use.