Skip to content

Apache DataFusion SQL Query Engine

Improve ExternalSorter ResourcesExhausted Error Message (#20226)

Eren Avsarogullari4d06c40d8d76
diff --git a/datafusion-cli/tests/snapshots/cli_top_memory_consumers@no_track.snap b/datafusion-cli/tests/snapshots/cli_top_memory_consumers@no_track.snap
index 89b646a531..fe454595eb 100644
--- a/datafusion-cli/tests/snapshots/cli_top_memory_consumers@no_track.snap
+++ b/datafusion-cli/tests/snapshots/cli_top_memory_consumers@no_track.snap
@@ -14,7 +14,7 @@ success: false
exit_code: 1
----- stdout -----
[CLI_VERSION]
-Error: Not enough memory to continue external sort. Consider increasing the memory limit, or decreasing sort_spill_reservation_bytes
+Error: Not enough memory to continue external sort. Consider increasing the memory limit config: 'datafusion.runtime.memory_limit', or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'.
caused by
Resources exhausted: Failed to allocate
diff --git a/datafusion-cli/tests/snapshots/[email protected] b/datafusion-cli/tests/snapshots/[email protected]
index 62f864b3ad..bb30e38716 100644
--- a/datafusion-cli/tests/snapshots/[email protected]
+++ b/datafusion-cli/tests/snapshots/[email protected]
@@ -14,7 +14,7 @@ success: false
exit_code: 1
----- stdout -----
[CLI_VERSION]
-Error: Not enough memory to continue external sort. Consider increasing the memory limit, or decreasing sort_spill_reservation_bytes
+Error: Not enough memory to continue external sort. Consider increasing the memory limit config: 'datafusion.runtime.memory_limit', or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'.
caused by
Resources exhausted: Additional allocation failed for ExternalSorter[0] with top memory consumers (across reservations) as:
Consumer(can spill: bool) consumed XB, peak XB,
diff --git a/datafusion-cli/tests/snapshots/cli_top_memory_consumers@top3_default.snap b/datafusion-cli/tests/snapshots/cli_top_memory_consumers@top3_default.snap
index 9845d095c9..891d72e3cc 100644
--- a/datafusion-cli/tests/snapshots/cli_top_memory_consumers@top3_default.snap
+++ b/datafusion-cli/tests/snapshots/cli_top_memory_consumers@top3_default.snap
@@ -12,7 +12,7 @@ success: false
exit_code: 1
----- stdout -----
[CLI_VERSION]
-Error: Not enough memory to continue external sort. Consider increasing the memory limit, or decreasing sort_spill_reservation_bytes
+Error: Not enough memory to continue external sort. Consider increasing the memory limit config: 'datafusion.runtime.memory_limit', or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'.
caused by
Resources exhausted: Additional allocation failed for ExternalSorter[0] with top memory consumers (across reservations) as:
Consumer(can spill: bool) consumed XB, peak XB,
diff --git a/datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rs
index 8d6e5dd7e4..af3031c690 100644
--- a/datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rs
+++ b/datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rs
@@ -110,7 +110,8 @@ async fn automatic_usage_example() -> Result<()> {
println!("✓ Expected memory limit error during data processing:");
println!("Error: {e}");
/* Example error message:
- Error: Not enough memory to continue external sort. Consider increasing the memory limit, or decreasing sort_spill_reservation_bytes
+ Error: Not enough memory to continue external sort. Consider increasing the memory limit config: 'datafusion.runtime.memory_limit',
+ or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'.
caused by
Resources exhausted: Additional allocation failed with top memory consumers (across reservations) as:
ExternalSorterMerge[3]#112(can spill: false) consumed 10.0 MB, peak 10.0 MB,
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs
index a8361f7b29..3432f9c478 100644
--- a/datafusion/physical-plan/src/sorts/sort.rs
+++ b/datafusion/physical-plan/src/sorts/sort.rs
@@ -819,7 +819,8 @@ impl ExternalSorter {
match e {
DataFusionError::ResourcesExhausted(_) => e.context(
"Not enough memory to continue external sort. \
- Consider increasing the memory limit, or decreasing sort_spill_reservation_bytes"
+ Consider increasing the memory limit config: 'datafusion.runtime.memory_limit', \
+ or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'."
),
// This is not an OOM error, so just return it as is.
_ => e,
@@ -1736,6 +1737,21 @@ mod tests {
"Assertion failed: expected a ResourcesExhausted error, but got: {err:?}"
);
+ // Verify external sorter error message when resource is exhausted
+ let config_vector = vec![
+ "datafusion.runtime.memory_limit",
+ "datafusion.execution.sort_spill_reservation_bytes",
+ ];
+ let error_message = err.message().to_string();
+ for config in config_vector.into_iter() {
+ assert!(
+ error_message.as_str().contains(config),
+ "Config: '{}' should be contained in error message: {}.",
+ config,
+ error_message.as_str()
+ );
+ }
+
Ok(())
}
@@ -1756,7 +1772,7 @@ mod tests {
// The input has 200 partitions, each partition has a batch containing 100 rows.
// Each row has a single Utf8 column, the Utf8 string values are roughly 42 bytes.
- // The total size of the input is roughly 8.4 KB.
+ // The total size of the input is roughly 820 KB.
let input = test::scan_partitioned_utf8(200);
let schema = input.schema();