Apache DataFusion SQL Query Engine
test: Add `datafusion-cli` `fair` and `unbounded` memory-pool test coverage (#20565)
Eren Avsarogullari8fe926d3871a
diff --git a/datafusion-cli/tests/cli_integration.rs b/datafusion-cli/tests/cli_integration.rsindex 99fc2d527e..7bc45693a8 100644--- a/datafusion-cli/tests/cli_integration.rs+++ b/datafusion-cli/tests/cli_integration.rs@@ -20,6 +20,7 @@ use std::process::Command;use rstest::rstest;use async_trait::async_trait;+use insta::internals::SettingsBindDropGuard;use insta::{Settings, glob};use insta_cmd::{assert_cmd_snapshot, get_cargo_bin};use std::path::PathBuf;@@ -215,6 +216,42 @@ fn test_cli_top_memory_consumers<'a>(#[case] snapshot_name: &str,#[case] top_memory_consumers: impl IntoIterator<Item = &'a str>,) {+ let _bound = bind_to_settings(snapshot_name);++ let mut cmd = cli();+ let sql = "select * from generate_series(1,500000) as t1(v1) order by v1;";+ cmd.args(["--memory-limit", "10M", "--command", sql]);+ cmd.args(top_memory_consumers);++ assert_cmd_snapshot!(cmd);+}++#[rstest]+#[case("no_track", ["--top-memory-consumers", "0"])]+#[case("top2", ["--top-memory-consumers", "2"])]+#[test]+fn test_cli_top_memory_consumers_with_mem_pool_type<'a>(+ #[case] snapshot_name: &str,+ #[case] top_memory_consumers: impl IntoIterator<Item = &'a str>,+) {+ let _bound = bind_to_settings(snapshot_name);++ let mut cmd = cli();+ let sql = "select * from generate_series(1,500000) as t1(v1) order by v1;";+ cmd.args([+ "--memory-limit",+ "10M",+ "--mem-pool-type",+ "fair",+ "--command",+ sql,+ ]);+ cmd.args(top_memory_consumers);++ assert_cmd_snapshot!(cmd);+}++fn bind_to_settings(snapshot_name: &str) -> SettingsBindDropGuard {let mut settings = make_settings();settings.set_snapshot_suffix(snapshot_name);@@ -232,12 +269,20 @@ fn test_cli_top_memory_consumers<'a>("Resources exhausted: Failed to allocate",);+ settings.bind_to_scope()+}++#[test]+fn test_cli_with_unbounded_memory_pool() {+ let mut settings = make_settings();++ settings.set_snapshot_suffix("default");+let _bound = settings.bind_to_scope();let mut cmd = cli();let sql = "select * from generate_series(1,500000) as t1(v1) order by v1;";- cmd.args(["--memory-limit", "10M", "--command", sql]);- cmd.args(top_memory_consumers);+ cmd.args(["--maxrows", "10", "--command", sql]);assert_cmd_snapshot!(cmd);}diff --git a/datafusion-cli/tests/snapshots/cli_top_memory_consumers_with_mem_pool_type@no_track.snap b/datafusion-cli/tests/snapshots/cli_top_memory_consumers_with_mem_pool_type@no_track.snapnew file mode 100644index 0000000000..25267ea161--- /dev/null+++ b/datafusion-cli/tests/snapshots/cli_top_memory_consumers_with_mem_pool_type@no_track.snap@@ -0,0 +1,23 @@+---+source: datafusion-cli/tests/cli_integration.rs+info:+ program: datafusion-cli+ args:+ - "--memory-limit"+ - 10M+ - "--mem-pool-type"+ - fair+ - "--command"+ - "select * from generate_series(1,500000) as t1(v1) order by v1;"+ - "--top-memory-consumers"+ - "0"+---+success: false+exit_code: 1+----- stdout -----+[CLI_VERSION]+Error: Not enough memory to continue external sort. Consider increasing the memory limit config: 'datafusion.runtime.memory_limit', or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'.+caused by+Resources exhausted: Failed to allocate++----- stderr -----diff --git a/datafusion-cli/tests/snapshots/[email protected] b/datafusion-cli/tests/snapshots/[email protected]new file mode 100644index 0000000000..6515050047--- /dev/null+++ b/datafusion-cli/tests/snapshots/[email protected]@@ -0,0 +1,26 @@+---+source: datafusion-cli/tests/cli_integration.rs+info:+ program: datafusion-cli+ args:+ - "--memory-limit"+ - 10M+ - "--mem-pool-type"+ - fair+ - "--command"+ - "select * from generate_series(1,500000) as t1(v1) order by v1;"+ - "--top-memory-consumers"+ - "2"+---+success: false+exit_code: 1+----- stdout -----+[CLI_VERSION]+Error: Not enough memory to continue external sort. Consider increasing the memory limit config: 'datafusion.runtime.memory_limit', or decreasing the config: 'datafusion.execution.sort_spill_reservation_bytes'.+caused by+Resources exhausted: Additional allocation failed for ExternalSorter[0] with top memory consumers (across reservations) as:+ Consumer(can spill: bool) consumed XB, peak XB,+ Consumer(can spill: bool) consumed XB, peak XB.+Error: Failed to allocate++----- stderr -----diff --git a/datafusion-cli/tests/snapshots/[email protected] b/datafusion-cli/tests/snapshots/[email protected]new file mode 100644index 0000000000..7bdcd63dc7--- /dev/null+++ b/datafusion-cli/tests/snapshots/[email protected]@@ -0,0 +1,36 @@+---+source: datafusion-cli/tests/cli_integration.rs+info:+ program: datafusion-cli+ args:+ - "--maxrows"+ - "10"+ - "--command"+ - "select * from generate_series(1,500000) as t1(v1) order by v1;"+---+success: true+exit_code: 0+----- stdout -----+[CLI_VERSION]++----++| v1 |++----++| 1 |+| 2 |+| 3 |+| 4 |+| 5 |+| 6 |+| 7 |+| 8 |+| 9 |+| 10 |+| . |+| . |+| . |++----++500000 row(s) fetched. (First 10 displayed. Use --maxrows to adjust)+[ELAPSED]+++----- stderr -----