Skip to content

Apache DataFusion SQL Query Engine

fix(cli): preserve rows in non-table (`csv`, `json`, etc) output (#26755)

Parafee417a260fafd317
diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rs
index cb09bb64a0..bc7cb01306 100644
--- a/datafusion-cli/src/exec.rs
+++ b/datafusion-cli/src/exec.rs
@@ -253,6 +253,7 @@ impl StatementExecutor {
let physical_plan = df.create_physical_plan().await?;
let task_ctx = ctx.task_ctx();
let options = task_ctx.session_config().options();
+ let print_options = adjusted.into_inner();
// Track memory usage for the query result if it's bounded
let reservation =
@@ -272,14 +273,15 @@ impl StatementExecutor {
.print_stream(stream, now, &options.format)
.await?;
} else {
- // Bounded stream; collected results size is limited by the maxrows option
+ // Non-table formats write every row, so only limit collection when the
+ // selected output format applies the maxrows option.
let schema = physical_plan.schema();
let mut stream = execute_stream(physical_plan, task_ctx.clone())?;
let mut results = vec![];
let mut row_count = 0_usize;
- let max_rows = match print_options.maxrows {
- MaxRows::Unlimited => usize::MAX,
- MaxRows::Limited(n) => n,
+ let max_rows = match (print_options.format, print_options.maxrows) {
+ (PrintFormat::Table, MaxRows::Limited(n)) => n,
+ _ => usize::MAX,
};
while let Some(batch) = stream.next().await {
let batch = batch?;
@@ -293,7 +295,7 @@ impl StatementExecutor {
}
row_count += curr_num_rows;
}
- adjusted.into_inner().print_batches(
+ print_options.print_batches(
schema,
&results,
now,
diff --git a/datafusion-cli/tests/cli_integration.rs b/datafusion-cli/tests/cli_integration.rs
index 80dc5f560c..4106885e6b 100644
--- a/datafusion-cli/tests/cli_integration.rs
+++ b/datafusion-cli/tests/cli_integration.rs
@@ -706,6 +706,38 @@ fn test_cli_wide_result_set_no_crash() {
assert_cmd_snapshot!(cmd);
}
+#[rstest]
+#[case("csv", "\n8200\n")]
+#[case("tsv", "\n8200\n")]
+#[case("json", r#"{"value":8200}]"#)]
+#[case("nd-json", "\n{\"value\":8200}\n")]
+fn test_non_table_formats_ignore_maxrows(
+ #[case] format: &str,
+ #[case] final_row_fragment: &str,
+) {
+ let output = cli()
+ .args([
+ "--format",
+ format,
+ "--maxrows",
+ "1",
+ "--command",
+ "SELECT value FROM generate_series(1, 8200)",
+ ])
+ .output()
+ .expect("Failed to execute command");
+
+ let stdout = String::from_utf8_lossy(&output.stdout);
+ assert!(
+ output.status.success(),
+ "query failed for {format} output:\n{stdout}"
+ );
+ assert!(
+ stdout.contains(final_row_fragment),
+ "expected the final row in {format} output"
+ );
+}
+
#[tokio::test]
async fn test_cli() {
let Some(container) = start_rustfs_or_skip().await else {