Skip to content

Apache DataFusion SQL Query Engine

fix: tree-style explain for memory table shows wrong row count (#24511)

Yongting You10077e8207af
diff --git a/datafusion/datasource/src/memory.rs b/datafusion/datasource/src/memory.rs
index 2370ed87a2..a4f51c28f8 100644
--- a/datafusion/datasource/src/memory.rs
+++ b/datafusion/datasource/src/memory.rs
@@ -129,7 +129,12 @@ impl DataSource for MemorySourceConfig {
}
}
DisplayFormatType::TreeRender => {
- let total_rows = self.partitions.iter().map(|b| b.len()).sum::<usize>();
+ let total_rows = self
+ .partitions
+ .iter()
+ .flatten()
+ .map(RecordBatch::num_rows)
+ .sum::<usize>();
let total_bytes: usize = self
.partitions
.iter()
diff --git a/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt b/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt
index 3dbf880fd1..79c6cb7a51 100644
--- a/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt
+++ b/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt
@@ -209,7 +209,7 @@ FROM (
100000 5000050000
# Assert spill happened in the final aggregation.
-# In multi-partitions configuration, 'spilled_rows' is not deterministic, so assert
+# In multi-partitions configuration, 'spilled_rows' is not deterministic, so assert
# the unit to be 'K'
query TT
EXPLAIN ANALYZE
diff --git a/datafusion/sqllogictest/test_files/array/init_data.slt.part b/datafusion/sqllogictest/test_files/array/init_data.slt.part
index bb8d76809f..06d0293030 100644
--- a/datafusion/sqllogictest/test_files/array/init_data.slt.part
+++ b/datafusion/sqllogictest/test_files/array/init_data.slt.part
@@ -688,4 +688,3 @@ AS
arrow_cast(column4, 'FixedSizeList(3, Float64)') AS column4
FROM arrays_distance_table
;
-
diff --git a/datafusion/sqllogictest/test_files/array_add.slt b/datafusion/sqllogictest/test_files/array_add.slt
index e13f6acd26..acef9984fd 100644
--- a/datafusion/sqllogictest/test_files/array_add.slt
+++ b/datafusion/sqllogictest/test_files/array_add.slt
@@ -234,4 +234,4 @@ select array_add(array_add(a, b), c) from (values
----
[111.0, 222.0]
NULL
-[111.0, NULL]
\ No newline at end of file
+[111.0, NULL]
diff --git a/datafusion/sqllogictest/test_files/array_product.slt b/datafusion/sqllogictest/test_files/array_product.slt
index ba60360d1c..a904282358 100644
--- a/datafusion/sqllogictest/test_files/array_product.slt
+++ b/datafusion/sqllogictest/test_files/array_product.slt
@@ -142,4 +142,4 @@ select list_product(column1) from (values
) as t(column1);
----
6
-NULL
\ No newline at end of file
+NULL
diff --git a/datafusion/sqllogictest/test_files/array_subtract.slt b/datafusion/sqllogictest/test_files/array_subtract.slt
index 4a680c93aa..28edd6ba9b 100644
--- a/datafusion/sqllogictest/test_files/array_subtract.slt
+++ b/datafusion/sqllogictest/test_files/array_subtract.slt
@@ -234,4 +234,4 @@ select array_subtract(array_subtract(a, b), c) from (values
----
[89.0, 178.0]
NULL
-[89.0, NULL]
\ No newline at end of file
+[89.0, NULL]
diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt
index 4e0397bb41..fcac86aa21 100644
--- a/datafusion/sqllogictest/test_files/explain_tree.slt
+++ b/datafusion/sqllogictest/test_files/explain_tree.slt
@@ -270,7 +270,7 @@ physical_plan
08)│ -------------------- │
09)│ bytes: 1024 │
10)│ format: memory │
-11)│ rows: 2 │
+11)│ rows: 6 │
12)└───────────────────────────┘
# 2 Joins
@@ -331,7 +331,7 @@ physical_plan
09)│ -------------------- ││ -------------------- │
10)│ bytes: 512 ││ on: ├──────────────┐
11)│ format: memory ││ (int_col = int_col) │ │
-12)│ rows: 1 ││ │ │
+12)│ rows: 3 ││ │ │
13)└───────────────────────────┘└─────────────┬─────────────┘ │
14)-----------------------------┌─────────────┴─────────────┐┌─────────────┴─────────────┐
15)-----------------------------│ DataSourceExec ││ RepartitionExec │
@@ -570,7 +570,7 @@ physical_plan
09)│ -------------------- │
10)│ bytes: 512 │
11)│ format: memory │
-12)│ rows: 1 │
+12)│ rows: 3 │
13)└───────────────────────────┘
# Query with filter on json
@@ -927,7 +927,7 @@ physical_plan
15)│ -------------------- │
16)│ bytes: 512 │
17)│ format: memory │
-18)│ rows: 1 │
+18)│ rows: 3 │
19)└───────────────────────────┘
# Query with projection on json
@@ -1177,7 +1177,7 @@ physical_plan
13)│ -------------------- ││ -------------------- │
14)│ bytes: 5932 ││ bytes: 5932 │
15)│ format: memory ││ format: memory │
-16)│ rows: 1 ││ rows: 1 │
+16)│ rows: 4 ││ rows: 4 │
17)└───────────────────────────┘└───────────────────────────┘
statement ok
@@ -1276,7 +1276,7 @@ physical_plan
44)│ -------------------- ││ -------------------- │
45)│ bytes: 288 ││ bytes: 280 │
46)│ format: memory ││ format: memory │
-47)│ rows: 1 ││ rows: 1 │
+47)│ rows: 3 ││ rows: 3 │
48)└───────────────────────────┘└───────────────────────────┘
# Test explain tree for UnionExec
@@ -1295,14 +1295,14 @@ physical_plan
06)│ -------------------- ││ -------------------- │
07)│ bytes: 288 ││ id: CAST(id AS Int32) │
08)│ format: memory ││ name: name │
-09)│ rows: 1 ││ │
+09)│ rows: 3 ││ │
10)└───────────────────────────┘└─────────────┬─────────────┘
11)-----------------------------┌─────────────┴─────────────┐
12)-----------------------------│ DataSourceExec │
13)-----------------------------│ -------------------- │
14)-----------------------------│ bytes: 280 │
15)-----------------------------│ format: memory │
-16)-----------------------------│ rows: 1 │
+16)-----------------------------│ rows: 3 │
17)-----------------------------└───────────────────────────┘
# cleanup
@@ -1646,7 +1646,7 @@ physical_plan
13)│ -------------------- │
14)│ bytes: 2576 │
15)│ format: memory │
-16)│ rows: 1 │
+16)│ rows: 3 │
17)└───────────────────────────┘
query TT
@@ -1669,7 +1669,7 @@ physical_plan
13)│ -------------------- │
14)│ bytes: 2576 │
15)│ format: memory │
-16)│ rows: 1 │
+16)│ rows: 3 │
17)└───────────────────────────┘
query TT
@@ -1692,7 +1692,7 @@ physical_plan
13)│ -------------------- │
14)│ bytes: 2576 │
15)│ format: memory │
-16)│ rows: 1 │
+16)│ rows: 3 │
17)└───────────────────────────┘
@@ -1754,7 +1754,7 @@ physical_plan
48)│ -------------------- │
49)│ bytes: 160 │
50)│ format: memory │
-51)│ rows: 1 │
+51)│ rows: 10 │
52)└───────────────────────────┘
# clean up
diff --git a/datafusion/sqllogictest/test_files/insert.slt b/datafusion/sqllogictest/test_files/insert.slt
index c807f73d60..ddb954bcfe 100644
--- a/datafusion/sqllogictest/test_files/insert.slt
+++ b/datafusion/sqllogictest/test_files/insert.slt
@@ -463,7 +463,7 @@ drop table unsigned_bigint_test
# Config reset
-# The SLT runner sets `target_partitions` to 4 instead of using the default, so
+# The SLT runner sets `target_partitions` to 4 instead of using the default, so
# reset it explicitly.
statement ok
set datafusion.execution.target_partitions = 4;
diff --git a/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt b/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt
index 1b6f2e4c86..eaafd70588 100644
--- a/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt
+++ b/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt
@@ -113,7 +113,7 @@ physical_plan
17)│ -------------------- ││ -------------------- │
18)│ bytes: 288 ││ bytes: 288 │
19)│ format: memory ││ format: memory │
-20)│ rows: 1 ││ rows: 1 │
+20)│ rows: 4 ││ rows: 4 │
21)└───────────────────────────┘└───────────────────────────┘
statement ok
diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt
index 58a655c02b..c1e4b054c3 100644
--- a/datafusion/sqllogictest/test_files/limit.slt
+++ b/datafusion/sqllogictest/test_files/limit.slt
@@ -401,7 +401,7 @@ physical_plan
09)│ -------------------- │
10)│ bytes: 160 │
11)│ format: memory │
-12)│ rows: 1 │
+12)│ rows: 10 │
13)└───────────────────────────┘
# generate BIGINT data from 1 to 1000 in multiple partitions
diff --git a/datafusion/sqllogictest/test_files/monotonic_projection_test.slt b/datafusion/sqllogictest/test_files/monotonic_projection_test.slt
index 71e5fbc08e..92e3744d40 100644
--- a/datafusion/sqllogictest/test_files/monotonic_projection_test.slt
+++ b/datafusion/sqllogictest/test_files/monotonic_projection_test.slt
@@ -314,4 +314,3 @@ DROP TABLE t2_int_float;
statement ok
DROP TABLE t3_int_bigint;
-
diff --git a/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt b/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt
index 2c53c94144..67fa20101d 100644
--- a/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt
+++ b/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt
@@ -73,7 +73,7 @@ GROUP BY v1
# ==================================================================================
# Input is partially ordered by group keys (input order by (a), query is 'group by a,b')
-#
+#
# Try different memory limits, ensure result is the same, but spill count differ
# HACK: check `spilled_bytes=x KB` to ensure it has spilled. If it has not spilled,
diff --git a/datafusion/sqllogictest/test_files/repartition.slt b/datafusion/sqllogictest/test_files/repartition.slt
index cf913caefc..7cec3b5844 100644
--- a/datafusion/sqllogictest/test_files/repartition.slt
+++ b/datafusion/sqllogictest/test_files/repartition.slt
@@ -140,7 +140,7 @@ FROM t1 WHERE ((false > (v1 = v1)) IS DISTINCT FROM true);
# Config reset
-# The SLT runner sets `target_partitions` to 4 instead of using the default, so
+# The SLT runner sets `target_partitions` to 4 instead of using the default, so
# reset it explicitly.
statement ok
set datafusion.execution.target_partitions = 4;
diff --git a/datafusion/sqllogictest/test_files/spark/math/hypot.slt b/datafusion/sqllogictest/test_files/spark/math/hypot.slt
index 564b34add8..519383d023 100644
--- a/datafusion/sqllogictest/test_files/spark/math/hypot.slt
+++ b/datafusion/sqllogictest/test_files/spark/math/hypot.slt
@@ -132,4 +132,4 @@ NaN
query R
SELECT hypot('Infinity'::double, '-Infinity'::double);
----
-Infinity
\ No newline at end of file
+Infinity
diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.slt
index a0e47a8691..941bab7c2d 100644
--- a/datafusion/sqllogictest/test_files/struct.slt
+++ b/datafusion/sqllogictest/test_files/struct.slt
@@ -1732,4 +1732,3 @@ drop view leaf_view;
statement ok
drop table leaf_base;
-
diff --git a/datafusion/sqllogictest/test_files/window_topn.slt b/datafusion/sqllogictest/test_files/window_topn.slt
index 8f65e496b5..a50b2d9e7e 100644
--- a/datafusion/sqllogictest/test_files/window_topn.slt
+++ b/datafusion/sqllogictest/test_files/window_topn.slt
@@ -283,7 +283,7 @@ physical_plan
54)│ -------------------- │
55)│ bytes: 480 │
56)│ format: memory │
-57)│ rows: 1 │
+57)│ rows: 10 │
58)└───────────────────────────┘
statement ok