Skip to content

Apache DataFusion SQL Query Engine

test: add sqllogictest coverage for DISTINCT / GROUP BY / aggregation on map columns (#23406)

Prateek Ganigi761f7f691894
diff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.slt
index 970ae2707d..f8cbb395cb 100644
--- a/datafusion/sqllogictest/test_files/map.slt
+++ b/datafusion/sqllogictest/test_files/map.slt
@@ -916,3 +916,135 @@ SELECT map([column1, column1 * 10], ['x','y']) FROM (VALUES (1), (2), (3)) t;
{1: x, 10: y}
{2: x, 20: y}
{3: x, 30: y}
+
+# tests for DISTINCT / GROUP BY / aggregation on map columns
+# https://github.com/apache/datafusion/issues/15428
+
+statement ok
+CREATE TABLE map_distinct_table AS VALUES
+ (MAP {'k1': 1, 'k2': 2}, 'a', 1),
+ (MAP {'k1': 1, 'k2': 2}, 'a', 2),
+ (MAP {'k1': 1, 'k2': 2}, 'b', 3),
+ (MAP {'k1': 3}, 'a', 4),
+ (MAP {'k1': CAST(NULL AS BIGINT)}, 'b', 5);
+
+statement ok
+INSERT INTO map_distinct_table VALUES (NULL, 'a', 6), (NULL, 'a', 7), (NULL, 'b', 8);
+
+# distinct on a map column collapses duplicate maps and duplicate NULLs
+query ? rowsort
+SELECT DISTINCT column1 FROM map_distinct_table;
+----
+NULL
+{k1: 1, k2: 2}
+{k1: 3}
+{k1: NULL}
+
+# exact reproducer from #15428: DISTINCT on a map column with LIMIT
+query ? rowsort
+SELECT DISTINCT column1 FROM map_distinct_table LIMIT 10;
+----
+NULL
+{k1: 1, k2: 2}
+{k1: 3}
+{k1: NULL}
+
+# distinct over a map column together with a scalar column
+query ?T rowsort
+SELECT DISTINCT column1, column2 FROM map_distinct_table;
+----
+NULL a
+NULL b
+{k1: 1, k2: 2} a
+{k1: 1, k2: 2} b
+{k1: 3} a
+{k1: NULL} b
+
+# group by a map column
+query ?II rowsort
+SELECT column1, COUNT(*), SUM(column3) FROM map_distinct_table GROUP BY column1;
+----
+NULL 3 21
+{k1: 1, k2: 2} 3 6
+{k1: 3} 1 4
+{k1: NULL} 1 5
+
+# group by a map column and a scalar column
+query ?TI rowsort
+SELECT column1, column2, COUNT(*) FROM map_distinct_table GROUP BY column1, column2;
+----
+NULL a 2
+NULL b 1
+{k1: 1, k2: 2} a 2
+{k1: 1, k2: 2} b 1
+{k1: 3} a 1
+{k1: NULL} b 1
+
+# empty maps compare equal under DISTINCT and are distinct from NULL
+query ? rowsort
+SELECT DISTINCT column1 FROM (VALUES (MAP {}), (MAP {}), (NULL)) t(column1);
+----
+NULL
+{}
+
+# HAVING clause with a map grouping key
+query ?I rowsort
+SELECT column1, COUNT(*) FROM map_distinct_table GROUP BY column1 HAVING COUNT(*) > 1;
+----
+NULL 3
+{k1: 1, k2: 2} 3
+
+# count and count distinct on a map column
+query II
+SELECT COUNT(column1), COUNT(DISTINCT column1) FROM map_distinct_table;
+----
+5 3
+
+# map column as input to an aggregate function
+query T?
+SELECT column2, array_agg(column1 ORDER BY column3) FROM map_distinct_table GROUP BY column2 ORDER BY column2;
+----
+a [{k1: 1, k2: 2}, {k1: 1, k2: 2}, {k1: 3}, NULL, NULL]
+b [{k1: 1, k2: 2}, {k1: NULL}, NULL]
+
+# UNION (distinct) on map columns
+query ?
+SELECT MAP {'a': 1} UNION SELECT MAP {'a': 1};
+----
+{a: 1}
+
+# unsorted maps are compared by entry order: maps with the same entries in a
+# different order are treated as distinct values
+query ? rowsort
+SELECT DISTINCT column1 FROM (SELECT MAP {'k1': 1, 'k2': 2} AS column1 UNION ALL SELECT MAP {'k2': 2, 'k1': 1});
+----
+{k1: 1, k2: 2}
+{k2: 2, k1: 1}
+
+statement ok
+DROP TABLE map_distinct_table;
+
+# distinct / group by on map columns read from parquet
+statement ok
+CREATE EXTERNAL TABLE map_data
+STORED AS PARQUET
+LOCATION '../core/tests/data/parquet_map.parquet';
+
+query I
+SELECT COUNT(*) FROM (SELECT DISTINCT ints, strings FROM map_data);
+----
+209
+
+query TI rowsort
+SELECT strings['method'] AS method, COUNT(*) FROM (SELECT DISTINCT strings FROM map_data) GROUP BY method;
+----
+DELETE 24
+GET 27
+HEAD 33
+OPTION 29
+PATCH 30
+POST 41
+PUT 25
+
+statement ok
+DROP TABLE map_data;