Skip to content

Apache DataFusion SQL Query Engine

refactor: move human readable display utilities to `datafusion-common` crate (#19080)

Yongting You482c6b8bd609
diff --git a/benchmarks/src/bin/external_aggr.rs b/benchmarks/src/bin/external_aggr.rs
index 46b6cc9a80..027ca07a8b 100644
--- a/benchmarks/src/bin/external_aggr.rs
+++ b/benchmarks/src/bin/external_aggr.rs
@@ -34,7 +34,6 @@ use datafusion::datasource::listing::{
use datafusion::datasource::{MemTable, TableProvider};
use datafusion::error::Result;
use datafusion::execution::memory_pool::FairSpillPool;
-use datafusion::execution::memory_pool::{human_readable_size, units};
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
use datafusion::execution::SessionStateBuilder;
use datafusion::physical_plan::display::DisplayableExecutionPlan;
@@ -44,6 +43,7 @@ use datafusion_benchmarks::util::{BenchmarkRun, CommonOpt, QueryResult};
use datafusion_common::instant::Instant;
use datafusion_common::utils::get_available_parallelism;
use datafusion_common::{exec_err, DEFAULT_PARQUET_EXTENSION};
+use datafusion_common::{human_readable_size, units};
#[derive(Debug, StructOpt)]
#[structopt(
diff --git a/benchmarks/src/util/memory.rs b/benchmarks/src/util/memory.rs
index 944239df31..11b96ef227 100644
--- a/benchmarks/src/util/memory.rs
+++ b/benchmarks/src/util/memory.rs
@@ -19,7 +19,7 @@
pub fn print_memory_stats() {
#[cfg(all(feature = "mimalloc", feature = "mimalloc_extended"))]
{
- use datafusion::execution::memory_pool::human_readable_size;
+ use datafusion_common::human_readable_size;
let mut peak_rss = 0;
let mut peak_commit = 0;
let mut page_faults = 0;
diff --git a/datafusion/common/src/display/human_readable.rs b/datafusion/common/src/display/human_readable.rs
new file mode 100644
index 0000000000..0e0d677bd8
--- /dev/null
+++ b/datafusion/common/src/display/human_readable.rs
@@ -0,0 +1,139 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Helpers for rendering sizes, counts, and durations in human readable form.
+
+/// Common data size units
+pub mod units {
+ pub const TB: u64 = 1 << 40;
+ pub const GB: u64 = 1 << 30;
+ pub const MB: u64 = 1 << 20;
+ pub const KB: u64 = 1 << 10;
+}
+
+/// Present size in human-readable form
+pub fn human_readable_size(size: usize) -> String {
+ use units::*;
+
+ let size = size as u64;
+ let (value, unit) = {
+ if size >= 2 * TB {
+ (size as f64 / TB as f64, "TB")
+ } else if size >= 2 * GB {
+ (size as f64 / GB as f64, "GB")
+ } else if size >= 2 * MB {
+ (size as f64 / MB as f64, "MB")
+ } else if size >= 2 * KB {
+ (size as f64 / KB as f64, "KB")
+ } else {
+ (size as f64, "B")
+ }
+ };
+ format!("{value:.1} {unit}")
+}
+
+/// Present count in human-readable form with K, M, B, T suffixes
+pub fn human_readable_count(count: usize) -> String {
+ let count = count as u64;
+ let (value, unit) = {
+ if count >= 1_000_000_000_000 {
+ (count as f64 / 1_000_000_000_000.0, " T")
+ } else if count >= 1_000_000_000 {
+ (count as f64 / 1_000_000_000.0, " B")
+ } else if count >= 1_000_000 {
+ (count as f64 / 1_000_000.0, " M")
+ } else if count >= 1_000 {
+ (count as f64 / 1_000.0, " K")
+ } else {
+ return count.to_string();
+ }
+ };
+
+ // Format with appropriate precision
+ // For values >= 100, show 1 decimal place (e.g., 123.4 K)
+ // For values < 100, show 2 decimal places (e.g., 10.12 K)
+ if value >= 100.0 {
+ format!("{value:.1}{unit}")
+ } else {
+ format!("{value:.2}{unit}")
+ }
+}
+
+/// Present duration in human-readable form with 2 decimal places
+pub fn human_readable_duration(nanos: u64) -> String {
+ const NANOS_PER_SEC: f64 = 1_000_000_000.0;
+ const NANOS_PER_MILLI: f64 = 1_000_000.0;
+ const NANOS_PER_MICRO: f64 = 1_000.0;
+
+ let nanos_f64 = nanos as f64;
+
+ if nanos >= 1_000_000_000 {
+ // >= 1 second: show in seconds
+ format!("{:.2}s", nanos_f64 / NANOS_PER_SEC)
+ } else if nanos >= 1_000_000 {
+ // >= 1 millisecond: show in milliseconds
+ format!("{:.2}ms", nanos_f64 / NANOS_PER_MILLI)
+ } else if nanos >= 1_000 {
+ // >= 1 microsecond: show in microseconds
+ format!("{:.2}µs", nanos_f64 / NANOS_PER_MICRO)
+ } else {
+ // < 1 microsecond: show in nanoseconds
+ format!("{nanos}ns")
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_human_readable_count() {
+ assert_eq!(human_readable_count(0), "0");
+ assert_eq!(human_readable_count(1), "1");
+ assert_eq!(human_readable_count(999), "999");
+ assert_eq!(human_readable_count(1_000), "1.00 K");
+ assert_eq!(human_readable_count(10_100), "10.10 K");
+ assert_eq!(human_readable_count(1_532), "1.53 K");
+ assert_eq!(human_readable_count(99_999), "100.00 K");
+ assert_eq!(human_readable_count(1_000_000), "1.00 M");
+ assert_eq!(human_readable_count(1_532_000), "1.53 M");
+ assert_eq!(human_readable_count(99_000_000), "99.00 M");
+ assert_eq!(human_readable_count(123_456_789), "123.5 M");
+ assert_eq!(human_readable_count(1_000_000_000), "1.00 B");
+ assert_eq!(human_readable_count(1_532_000_000), "1.53 B");
+ assert_eq!(human_readable_count(999_999_999_999), "1000.0 B");
+ assert_eq!(human_readable_count(1_000_000_000_000), "1.00 T");
+ assert_eq!(human_readable_count(42_000_000_000_000), "42.00 T");
+ }
+
+ #[test]
+ fn test_human_readable_duration() {
+ assert_eq!(human_readable_duration(0), "0ns");
+ assert_eq!(human_readable_duration(1), "1ns");
+ assert_eq!(human_readable_duration(999), "999ns");
+ assert_eq!(human_readable_duration(1_000), "1.00µs");
+ assert_eq!(human_readable_duration(1_234), "1.23µs");
+ assert_eq!(human_readable_duration(999_999), "1000.00µs");
+ assert_eq!(human_readable_duration(1_000_000), "1.00ms");
+ assert_eq!(human_readable_duration(11_295_377), "11.30ms");
+ assert_eq!(human_readable_duration(1_234_567), "1.23ms");
+ assert_eq!(human_readable_duration(999_999_999), "1000.00ms");
+ assert_eq!(human_readable_duration(1_000_000_000), "1.00s");
+ assert_eq!(human_readable_duration(1_234_567_890), "1.23s");
+ assert_eq!(human_readable_duration(42_000_000_000), "42.00s");
+ }
+}
diff --git a/datafusion/common/src/display/mod.rs b/datafusion/common/src/display/mod.rs
index bad51c45f8..a6a97b243f 100644
--- a/datafusion/common/src/display/mod.rs
+++ b/datafusion/common/src/display/mod.rs
@@ -18,6 +18,7 @@
//! Types for plan display
mod graphviz;
+pub mod human_readable;
pub use graphviz::*;
use std::{
diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs
index 763bd2d8b1..3bec9bd35c 100644
--- a/datafusion/common/src/lib.rs
+++ b/datafusion/common/src/lib.rs
@@ -69,6 +69,9 @@ pub use dfschema::{
DFSchema, DFSchemaRef, ExprSchema, SchemaExt, ToDFSchema, qualified_name,
};
pub use diagnostic::Diagnostic;
+pub use display::human_readable::{
+ human_readable_count, human_readable_duration, human_readable_size, units,
+};
pub use error::{
DataFusionError, Result, SchemaError, SharedResult, field_not_found,
unqualified_field_not_found,
diff --git a/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs b/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs
index 2ce7db3ea4..9c7e4f1b84 100644
--- a/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs
+++ b/datafusion/core/tests/fuzz_cases/sort_query_fuzz.rs
@@ -24,11 +24,9 @@ use arrow::array::RecordBatch;
use arrow_schema::SchemaRef;
use datafusion::datasource::MemTable;
use datafusion::prelude::{SessionConfig, SessionContext};
-use datafusion_common::{instant::Instant, Result};
+use datafusion_common::{human_readable_size, instant::Instant, Result};
use datafusion_execution::disk_manager::DiskManagerBuilder;
-use datafusion_execution::memory_pool::{
- human_readable_size, MemoryPool, UnboundedMemoryPool,
-};
+use datafusion_execution::memory_pool::{MemoryPool, UnboundedMemoryPool};
use datafusion_expr::display_schema;
use datafusion_physical_plan::spill::get_record_batch_memory_size;
use std::time::Duration;
diff --git a/datafusion/core/tests/fuzz_cases/spilling_fuzz_in_memory_constrained_env.rs b/datafusion/core/tests/fuzz_cases/spilling_fuzz_in_memory_constrained_env.rs
index 6c1bd316cd..6ea0182c25 100644
--- a/datafusion/core/tests/fuzz_cases/spilling_fuzz_in_memory_constrained_env.rs
+++ b/datafusion/core/tests/fuzz_cases/spilling_fuzz_in_memory_constrained_env.rs
@@ -31,7 +31,7 @@ use datafusion::physical_plan::expressions::PhysicalSortExpr;
use datafusion::physical_plan::sorts::sort::SortExec;
use datafusion::physical_plan::ExecutionPlan;
use datafusion::prelude::SessionConfig;
-use datafusion_execution::memory_pool::units::{KB, MB};
+use datafusion_common::units::{KB, MB};
use datafusion_execution::memory_pool::{
FairSpillPool, MemoryConsumer, MemoryReservation,
};
diff --git a/datafusion/core/tests/memory_limit/memory_limit_validation/utils.rs b/datafusion/core/tests/memory_limit/memory_limit_validation/utils.rs
index 7b157b707a..899c9f88d8 100644
--- a/datafusion/core/tests/memory_limit/memory_limit_validation/utils.rs
+++ b/datafusion/core/tests/memory_limit/memory_limit_validation/utils.rs
@@ -22,10 +22,8 @@ use sysinfo::{ProcessRefreshKind, ProcessesToUpdate, System};
use tokio::time::{interval, Duration};
use datafusion::prelude::{SessionConfig, SessionContext};
-use datafusion_execution::{
- memory_pool::{human_readable_size, FairSpillPool},
- runtime_env::RuntimeEnvBuilder,
-};
+use datafusion_common::human_readable_size;
+use datafusion_execution::{memory_pool::FairSpillPool, runtime_env::RuntimeEnvBuilder};
/// Measures the maximum RSS (in bytes) during the execution of an async task. RSS
/// will be sampled every 7ms.
diff --git a/datafusion/execution/src/disk_manager.rs b/datafusion/execution/src/disk_manager.rs
index 346dda5fd0..1272e6c5f8 100644
--- a/datafusion/execution/src/disk_manager.rs
+++ b/datafusion/execution/src/disk_manager.rs
@@ -28,7 +28,7 @@ use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use tempfile::{Builder, NamedTempFile, TempDir};
-use crate::memory_pool::human_readable_size;
+use datafusion_common::human_readable_size;
pub const DEFAULT_MAX_TEMP_DIRECTORY_SIZE: u64 = 100 * 1024 * 1024 * 1024; // 100GB
diff --git a/datafusion/execution/src/memory_pool/mod.rs b/datafusion/execution/src/memory_pool/mod.rs
index 5df7ec955d..fbf9ce41da 100644
--- a/datafusion/execution/src/memory_pool/mod.rs
+++ b/datafusion/execution/src/memory_pool/mod.rs
@@ -27,6 +27,9 @@ pub mod proxy {
pub use datafusion_common::utils::proxy::{HashTableAllocExt, VecAllocExt};
}
+pub use datafusion_common::{
+ human_readable_count, human_readable_duration, human_readable_size, units,
+};
pub use pool::*;
/// Tracks and potentially limits memory use across operators during execution.
@@ -473,84 +476,6 @@ impl Drop for MemoryReservation {
}
}
-pub mod units {
- pub const TB: u64 = 1 << 40;
- pub const GB: u64 = 1 << 30;
- pub const MB: u64 = 1 << 20;
- pub const KB: u64 = 1 << 10;
-}
-
-/// Present size in human-readable form
-pub fn human_readable_size(size: usize) -> String {
- use units::*;
-
- let size = size as u64;
- let (value, unit) = {
- if size >= 2 * TB {
- (size as f64 / TB as f64, "TB")
- } else if size >= 2 * GB {
- (size as f64 / GB as f64, "GB")
- } else if size >= 2 * MB {
- (size as f64 / MB as f64, "MB")
- } else if size >= 2 * KB {
- (size as f64 / KB as f64, "KB")
- } else {
- (size as f64, "B")
- }
- };
- format!("{value:.1} {unit}")
-}
-
-/// Present count in human-readable form with K, M, B, T suffixes
-pub fn human_readable_count(count: usize) -> String {
- let count = count as u64;
- let (value, unit) = {
- if count >= 1_000_000_000_000 {
- (count as f64 / 1_000_000_000_000.0, " T")
- } else if count >= 1_000_000_000 {
- (count as f64 / 1_000_000_000.0, " B")
- } else if count >= 1_000_000 {
- (count as f64 / 1_000_000.0, " M")
- } else if count >= 1_000 {
- (count as f64 / 1_000.0, " K")
- } else {
- return count.to_string();
- }
- };
-
- // Format with appropriate precision
- // For values >= 100, show 1 decimal place (e.g., 123.4 K)
- // For values < 100, show 2 decimal places (e.g., 10.12 K)
- if value >= 100.0 {
- format!("{value:.1}{unit}")
- } else {
- format!("{value:.2}{unit}")
- }
-}
-
-/// Present duration in human-readable form with 2 decimal places
-pub fn human_readable_duration(nanos: u64) -> String {
- const NANOS_PER_SEC: f64 = 1_000_000_000.0;
- const NANOS_PER_MILLI: f64 = 1_000_000.0;
- const NANOS_PER_MICRO: f64 = 1_000.0;
-
- let nanos_f64 = nanos as f64;
-
- if nanos >= 1_000_000_000 {
- // >= 1 second: show in seconds
- format!("{:.2}s", nanos_f64 / NANOS_PER_SEC)
- } else if nanos >= 1_000_000 {
- // >= 1 millisecond: show in milliseconds
- format!("{:.2}ms", nanos_f64 / NANOS_PER_MILLI)
- } else if nanos >= 1_000 {
- // >= 1 microsecond: show in microseconds
- format!("{:.2}µs", nanos_f64 / NANOS_PER_MICRO)
- } else {
- // < 1 microsecond: show in nanoseconds
- format!("{nanos}ns")
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
@@ -647,57 +572,4 @@ mod tests {
assert_eq!(r2.size(), 25);
assert_eq!(pool.reserved(), 28);
}
-
- #[test]
- fn test_human_readable_count() {
- // Test small numbers (< 1000) - should display as-is
- assert_eq!(human_readable_count(0), "0");
- assert_eq!(human_readable_count(1), "1");
- assert_eq!(human_readable_count(999), "999");
-
- // Test thousands (K)
- assert_eq!(human_readable_count(1_000), "1.00 K");
- assert_eq!(human_readable_count(10_100), "10.10 K");
- assert_eq!(human_readable_count(1_532), "1.53 K");
- assert_eq!(human_readable_count(99_999), "100.00 K");
-
- // Test millions (M)
- assert_eq!(human_readable_count(1_000_000), "1.00 M");
- assert_eq!(human_readable_count(1_532_000), "1.53 M");
- assert_eq!(human_readable_count(99_000_000), "99.00 M");
- assert_eq!(human_readable_count(123_456_789), "123.5 M");
-
- // Test billions (B)
- assert_eq!(human_readable_count(1_000_000_000), "1.00 B");
- assert_eq!(human_readable_count(1_532_000_000), "1.53 B");
- assert_eq!(human_readable_count(999_999_999_999), "1000.0 B");
-
- // Test trillions (T)
- assert_eq!(human_readable_count(1_000_000_000_000), "1.00 T");
- assert_eq!(human_readable_count(42_000_000_000_000), "42.00 T");
- }
-
- #[test]
- fn test_human_readable_duration() {
- // Test nanoseconds (< 1µs)
- assert_eq!(human_readable_duration(0), "0ns");
- assert_eq!(human_readable_duration(1), "1ns");
- assert_eq!(human_readable_duration(999), "999ns");
-
- // Test microseconds (1µs to < 1ms)
- assert_eq!(human_readable_duration(1_000), "1.00µs");
- assert_eq!(human_readable_duration(1_234), "1.23µs");
- assert_eq!(human_readable_duration(999_999), "1000.00µs");
-
- // Test milliseconds (1ms to < 1s)
- assert_eq!(human_readable_duration(1_000_000), "1.00ms");
- assert_eq!(human_readable_duration(11_295_377), "11.30ms");
- assert_eq!(human_readable_duration(1_234_567), "1.23ms");
- assert_eq!(human_readable_duration(999_999_999), "1000.00ms");
-
- // Test seconds (>= 1s)
- assert_eq!(human_readable_duration(1_000_000_000), "1.00s");
- assert_eq!(human_readable_duration(1_234_567_890), "1.23s");
- assert_eq!(human_readable_duration(42_000_000_000), "42.00s");
- }
}
diff --git a/datafusion/physical-plan/benches/spill_io.rs b/datafusion/physical-plan/benches/spill_io.rs
index 5123b91a6a..fbf190518d 100644
--- a/datafusion/physical-plan/benches/spill_io.rs
+++ b/datafusion/physical-plan/benches/spill_io.rs
@@ -25,8 +25,8 @@ use criterion::{
criterion_group, criterion_main, BatchSize, BenchmarkGroup, BenchmarkId, Criterion,
};
use datafusion_common::config::SpillCompression;
+use datafusion_common::human_readable_size;
use datafusion_common::instant::Instant;
-use datafusion_execution::memory_pool::human_readable_size;
use datafusion_execution::runtime_env::RuntimeEnv;
use datafusion_physical_plan::common::collect;
use datafusion_physical_plan::metrics::{ExecutionPlanMetricsSet, SpillMetrics};
diff --git a/datafusion/physical-plan/src/metrics/value.rs b/datafusion/physical-plan/src/metrics/value.rs
index d2710408d0..901494883a 100644
--- a/datafusion/physical-plan/src/metrics/value.rs
+++ b/datafusion/physical-plan/src/metrics/value.rs
@@ -20,7 +20,7 @@
use super::CustomMetricValue;
use chrono::{DateTime, Utc};
use datafusion_common::instant::Instant;
-use datafusion_execution::memory_pool::{
+use datafusion_common::{
human_readable_count, human_readable_duration, human_readable_size,
};
use parking_lot::Mutex;
@@ -1034,7 +1034,7 @@ mod tests {
use std::any::Any;
use chrono::TimeZone;
- use datafusion_execution::memory_pool::units::MB;
+ use datafusion_common::units::MB;
use super::*;
diff --git a/datafusion/physical-plan/src/sorts/streaming_merge.rs b/datafusion/physical-plan/src/sorts/streaming_merge.rs
index 047fbd8cbd..8003149a84 100644
--- a/datafusion/physical-plan/src/sorts/streaming_merge.rs
+++ b/datafusion/physical-plan/src/sorts/streaming_merge.rs
@@ -27,11 +27,11 @@ use crate::sorts::{
use crate::{SendableRecordBatchStream, SpillManager};
use arrow::array::*;
use arrow::datatypes::{DataType, SchemaRef};
+use datafusion_common::human_readable_size;
use datafusion_common::{assert_or_internal_err, internal_err, Result};
use datafusion_execution::disk_manager::RefCountedTempFile;
use datafusion_execution::memory_pool::{
- human_readable_size, MemoryConsumer, MemoryPool, MemoryReservation,
- UnboundedMemoryPool,
+ MemoryConsumer, MemoryPool, MemoryReservation, UnboundedMemoryPool,
};
use datafusion_physical_expr_common::sort_expr::LexOrdering;
use std::sync::Arc;