Apache DataFusion SQL Query Engine
Improve ExternalSorter ResourcesExhausted Error Message (#20226)
Eren Avsarogullari4d06c40d8d76
diff --git a/datafusion-cli/tests/snapshots/cli_top_memory_consumers@no_track.snap b/datafusion-cli/tests/snapshots/cli_top_memory_consumers@no_track.snapindex 89b646a531..fe454595eb 100644--- a/datafusion-cli/tests/snapshots/cli_top_memory_consumers@no_track.snap+++ b/datafusion-cli/tests/snapshots/cli_top_memory_consumers@no_track.snap@@ -14,7 +14,7 @@ success: falseexit_code: 1----- stdout -----[CLI_VERSION]-Error: Not enough memory to continue external sort. Consider increasing the memory limit, or decreasing sort_spill_reservation_bytes+Error: Not enough memory to continue external sort. Consider increasing the memory limit config: 'datafusion.runtime.memory_limit', or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'.caused byResources exhausted: Failed to allocatediff --git a/datafusion-cli/tests/snapshots/[email protected] b/datafusion-cli/tests/snapshots/[email protected]index 62f864b3ad..bb30e38716 100644--- a/datafusion-cli/tests/snapshots/[email protected]+++ b/datafusion-cli/tests/snapshots/[email protected]@@ -14,7 +14,7 @@ success: falseexit_code: 1----- stdout -----[CLI_VERSION]-Error: Not enough memory to continue external sort. Consider increasing the memory limit, or decreasing sort_spill_reservation_bytes+Error: Not enough memory to continue external sort. Consider increasing the memory limit config: 'datafusion.runtime.memory_limit', or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'.caused byResources exhausted: Additional allocation failed for ExternalSorter[0] with top memory consumers (across reservations) as:Consumer(can spill: bool) consumed XB, peak XB,diff --git a/datafusion-cli/tests/snapshots/cli_top_memory_consumers@top3_default.snap b/datafusion-cli/tests/snapshots/cli_top_memory_consumers@top3_default.snapindex 9845d095c9..891d72e3cc 100644--- a/datafusion-cli/tests/snapshots/cli_top_memory_consumers@top3_default.snap+++ b/datafusion-cli/tests/snapshots/cli_top_memory_consumers@top3_default.snap@@ -12,7 +12,7 @@ success: falseexit_code: 1----- stdout -----[CLI_VERSION]-Error: Not enough memory to continue external sort. Consider increasing the memory limit, or decreasing sort_spill_reservation_bytes+Error: Not enough memory to continue external sort. Consider increasing the memory limit config: 'datafusion.runtime.memory_limit', or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'.caused byResources exhausted: Additional allocation failed for ExternalSorter[0] with top memory consumers (across reservations) as:Consumer(can spill: bool) consumed XB, peak XB,diff --git a/datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rsindex 8d6e5dd7e4..af3031c690 100644--- a/datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rs+++ b/datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rs@@ -110,7 +110,8 @@ async fn automatic_usage_example() -> Result<()> {println!("✓ Expected memory limit error during data processing:");println!("Error: {e}");/* Example error message:- Error: Not enough memory to continue external sort. Consider increasing the memory limit, or decreasing sort_spill_reservation_bytes+ Error: Not enough memory to continue external sort. Consider increasing the memory limit config: 'datafusion.runtime.memory_limit',+ or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'.caused byResources exhausted: Additional allocation failed with top memory consumers (across reservations) as:ExternalSorterMerge[3]#112(can spill: false) consumed 10.0 MB, peak 10.0 MB,diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex a8361f7b29..3432f9c478 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -819,7 +819,8 @@ impl ExternalSorter {match e {DataFusionError::ResourcesExhausted(_) => e.context("Not enough memory to continue external sort. \- Consider increasing the memory limit, or decreasing sort_spill_reservation_bytes"+ Consider increasing the memory limit config: 'datafusion.runtime.memory_limit', \+ or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'."),// This is not an OOM error, so just return it as is._ => e,@@ -1736,6 +1737,21 @@ mod tests {"Assertion failed: expected a ResourcesExhausted error, but got: {err:?}");+ // Verify external sorter error message when resource is exhausted+ let config_vector = vec![+ "datafusion.runtime.memory_limit",+ "datafusion.execution.sort_spill_reservation_bytes",+ ];+ let error_message = err.message().to_string();+ for config in config_vector.into_iter() {+ assert!(+ error_message.as_str().contains(config),+ "Config: '{}' should be contained in error message: {}.",+ config,+ error_message.as_str()+ );+ }+Ok(())}@@ -1756,7 +1772,7 @@ mod tests {// The input has 200 partitions, each partition has a batch containing 100 rows.// Each row has a single Utf8 column, the Utf8 string values are roughly 42 bytes.- // The total size of the input is roughly 8.4 KB.+ // The total size of the input is roughly 820 KB.let input = test::scan_partitioned_utf8(200);let schema = input.schema();