Skip to content

Apache DataFusion SQL Query Engine

chore: Hide test-only internal public APIs (#25695)

Yongting You2b1fcae86640
diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rs
index 9cfa32e0d0..6fbff7dd5d 100644
--- a/datafusion/catalog/src/memory/table.rs
+++ b/datafusion/catalog/src/memory/table.rs
@@ -71,7 +71,12 @@ pub use datafusion_datasource::memory::PartitionData;
#[derive(Debug)]
pub struct MemTable {
schema: SchemaRef,
- // batches used to be pub(crate), but it's needed to be public for the tests
+ /// # Public Only for Internal Use:
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+ #[doc(hidden)]
pub batches: Vec<PartitionData>,
constraints: Constraints,
column_defaults: HashMap<String, Expr>,
diff --git a/datafusion/core/src/lib.rs b/datafusion/core/src/lib.rs
index 28ea4b4490..ff831a2676 100644
--- a/datafusion/core/src/lib.rs
+++ b/datafusion/core/src/lib.rs
@@ -855,7 +855,19 @@ pub mod physical_plan {
}
// Reexport testing macros for compatibility
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
pub use datafusion_common::assert_batches_eq;
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
pub use datafusion_common::assert_batches_sorted_eq;
/// re-export of [`datafusion_sql`] crate
@@ -895,10 +907,22 @@ pub mod variable {
pub use datafusion_expr::var_provider::{VarProvider, VarType};
}
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
#[cfg(not(target_arch = "wasm32"))]
+#[doc(hidden)]
pub mod test;
mod schema_equivalence;
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
pub mod test_util;
#[cfg(doctest)]
diff --git a/datafusion/datasource/src/mod.rs b/datafusion/datasource/src/mod.rs
index d0bf64d81c..6135835616 100644
--- a/datafusion/datasource/src/mod.rs
+++ b/datafusion/datasource/src/mod.rs
@@ -413,8 +413,6 @@ impl From<ObjectMeta> for PartitionedFile {
/// Generates test files with min-max statistics in different overlap patterns.
///
-/// Used by tests and benchmarks.
-///
/// # Overlap Factors
///
/// The `overlap_factor` parameter controls how much the value ranges in generated test files overlap:
@@ -450,6 +448,13 @@ impl From<ObjectMeta> for PartitionedFile {
/// File 2: [40, 140]
/// File 3: [60, 160]
/// File 4: [80, 180]
+///
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec<FileGroup> {
let mut files = Vec::with_capacity(num_files);
if num_files == 0 {
@@ -500,8 +505,14 @@ pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec<FileGro
vec![FileGroup::new(files)]
}
-// Helper function to verify that files within each group maintain sort order
-/// Used by tests and benchmarks
+/// Verifies that files within each group maintain sort order.
+///
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
pub fn verify_sort_integrity(file_groups: &[FileGroup]) -> bool {
for group in file_groups {
// Known-empty files contribute no rows and may not have min/max
diff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rs
index ed9b4df4ce..48ed38e8f4 100644
--- a/datafusion/expr/src/lib.rs
+++ b/datafusion/expr/src/lib.rs
@@ -88,6 +88,12 @@ pub mod preimage;
pub mod ptr_eq;
#[cfg(not(feature = "sql"))]
pub mod sql;
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
pub mod test;
pub mod tree_node;
pub mod type_coercion;
diff --git a/datafusion/functions-aggregate-common/src/tdigest.rs b/datafusion/functions-aggregate-common/src/tdigest.rs
index 860f56f6c0..ec9b988df2 100644
--- a/datafusion/functions-aggregate-common/src/tdigest.rs
+++ b/datafusion/functions-aggregate-common/src/tdigest.rs
@@ -206,7 +206,12 @@ impl TDigest {
v.clamp(min, max)
}
- // public for testing in other modules
+ /// # Public Only for Internal Use:
+ ///
+ /// This is not a public API and is for internal use only; see [API policy] for details.
+ ///
+ /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+ #[doc(hidden)]
pub fn merge_unsorted_f64(&self, unsorted_values: Vec<f64>) -> TDigest {
let mut values = unsorted_values;
values.sort_by(|a, b| a.total_cmp(b));
diff --git a/datafusion/physical-expr/src/intervals/mod.rs b/datafusion/physical-expr/src/intervals/mod.rs
index 9752ca27b5..86f8d2b463 100644
--- a/datafusion/physical-expr/src/intervals/mod.rs
+++ b/datafusion/physical-expr/src/intervals/mod.rs
@@ -18,5 +18,11 @@
//! Interval arithmetic and constraint propagation library
pub mod cp_solver;
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
pub mod test_utils;
pub mod utils;
diff --git a/datafusion/physical-plan/src/joins/hash_join/mod.rs b/datafusion/physical-plan/src/joins/hash_join/mod.rs
index 7b50435e18..aef27fa305 100644
--- a/datafusion/physical-plan/src/joins/hash_join/mod.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/mod.rs
@@ -18,7 +18,14 @@
//! [`HashJoinExec`] Partitioned Hash Join Operator
pub use exec::{HashJoinExec, HashJoinExecBuilder, PreparedHashJoinBuild};
-pub use partitioned_hash_eval::{HashExpr, HashTableLookupExpr, SeededRandomState};
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
+pub use partitioned_hash_eval::HashTableLookupExpr;
+pub use partitioned_hash_eval::{HashExpr, SeededRandomState};
mod exec;
mod inlist_builder;
diff --git a/datafusion/physical-plan/src/joins/mod.rs b/datafusion/physical-plan/src/joins/mod.rs
index 0d2480c131..8ffc6a9467 100644
--- a/datafusion/physical-plan/src/joins/mod.rs
+++ b/datafusion/physical-plan/src/joins/mod.rs
@@ -24,9 +24,15 @@ use arrow::array::BooleanBufferBuilder;
pub use asof_join::{AsOfJoinExec, AsOfMatchExpr};
pub use cross_join::CrossJoinExec;
use datafusion_physical_expr::PhysicalExprRef;
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
+pub use hash_join::HashTableLookupExpr;
pub use hash_join::{
- HashExpr, HashJoinExec, HashJoinExecBuilder, HashTableLookupExpr,
- PreparedHashJoinBuild, SeededRandomState,
+ HashExpr, HashJoinExec, HashJoinExecBuilder, PreparedHashJoinBuild, SeededRandomState,
};
pub use nested_loop_join::{NestedLoopJoinExec, NestedLoopJoinExecBuilder};
use parking_lot::Mutex;
@@ -52,8 +58,12 @@ mod array_map;
mod join_filter;
/// Hash map implementations for join operations.
///
-/// Note: This module is public for internal testing purposes only
-/// and is not guaranteed to be stable across versions.
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
pub mod join_hash_map;
use array_map::ArrayMap;
diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rs
index 5ff6cec374..40489eada3 100644
--- a/datafusion/physical-plan/src/lib.rs
+++ b/datafusion/physical-plan/src/lib.rs
@@ -115,4 +115,10 @@ pub mod udaf {
pub use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
}
+/// # Public Only for Internal Use:
+///
+/// This is not a public API and is for internal use only; see [API policy] for details.
+///
+/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis
+#[doc(hidden)]
pub mod test;
diff --git a/docs/source/contributor-guide/api-health.md b/docs/source/contributor-guide/api-health.md
index 8737df280f..e74d74ea51 100644
--- a/docs/source/contributor-guide/api-health.md
+++ b/docs/source/contributor-guide/api-health.md
@@ -43,6 +43,10 @@ Examples of non-breaking changes include:
- Marking a function as deprecated (`#[deprecated]`)
- Adding a new function to a `trait` with a default implementation
+<!-- Keep this anchor stable: it is linked from Rust API documentation. -->
+
+<a id="internal-public-apis"></a>
+
### DataFusion-internal Public APIs
Some internal implementation details require `pub` visibility because they are
@@ -55,6 +59,10 @@ notice or a deprecation period. Examples include:
2. Operator APIs required by the optimizer to inspect or rewrite execution plans
across crate boundaries.
+Do not expose internal APIs solely for tests or microbenchmarks. Some legacy code
+does so, but new tests and benchmarks should exercise observable behavior to
+simplify maintenance.
+
For APIs intended only for internal use, add `#[doc(hidden)]` and a doc comment
section headed `# Public Only for Internal Use:`. Name the crate or component
that requires access and explain why the API is not intended for downstream use.