Skip to content

Apache DataFusion SQL Query Engine

Keep ClickBench SQL in sync across runners (#25773)

kosiewb645de03f3d8
diff --git a/benchmarks/sql_benchmarks/clickbench_extended/benchmarks/q06.benchmark b/benchmarks/sql_benchmarks/clickbench_extended/benchmarks/q06.benchmark
index 4ae1a8efb6..f465e0dc6e 100644
--- a/benchmarks/sql_benchmarks/clickbench_extended/benchmarks/q06.benchmark
+++ b/benchmarks/sql_benchmarks/clickbench_extended/benchmarks/q06.benchmark
@@ -12,6 +12,6 @@ SELECT COUNT(*) > 0 from hits;
true
run
-SELECT COUNT(*) AS ShareCount FROM hits WHERE "IsMobile" = 1 AND "MobilePhoneModel" LIKE 'iPhone%' AND "SocialAction" = 'share' AND "SocialSourceNetworkID" IN (5, 12) AND "ClientTimeZone" BETWEEN -5 AND 5 AND regexp_match("Referer", '\/campaign\/(spring|summer)_promo') IS NOT NULL AND CASE WHEN split_part(split_part(CAST("URL" AS STRING), 'resolution=', 2), '&', 1) ~ '^\d+$' THEN split_part(split_part(CAST("URL" AS STRING), 'resolution=', 2), '&', 1)::INT ELSE 0 END > 1920 AND levenshtein(CAST("UTMSource" AS STRING), CAST("UTMCampaign" AS STRING)) < 3;
+SELECT COUNT(*) AS ShareCount FROM hits WHERE "IsMobile" = 1 AND "MobilePhoneModel" LIKE 'iPhone%' AND "SocialAction" = 'share' AND "SocialSourceNetworkID" IN (5, 12) AND "ClientTimeZone" BETWEEN -5 AND 5 AND regexp_match("Referer", '\/campaign\/(spring|summer)_promo') IS NOT NULL AND CASE WHEN split_part(split_part("URL", 'resolution=', 2), '&', 1) ~ '^\d+$' THEN split_part(split_part("URL", 'resolution=', 2), '&', 1)::INT ELSE 0 END > 1920 AND levenshtein(CAST("UTMSource" AS STRING), CAST("UTMCampaign" AS STRING)) < 3;
result sql_benchmarks/clickbench_extended/results/q06.csv
diff --git a/benchmarks/sql_benchmarks/clickbench_extended/benchmarks/q15.benchmark b/benchmarks/sql_benchmarks/clickbench_extended/benchmarks/q15.benchmark
new file mode 100644
index 0000000000..50ff902292
--- /dev/null
+++ b/benchmarks/sql_benchmarks/clickbench_extended/benchmarks/q15.benchmark
@@ -0,0 +1,21 @@
+name Q15
+group clickbench_extended
+subgroup ${CLICKBENCH_TYPE:-single}
+
+init sql_benchmarks/clickbench/init/set_config.sql
+
+load sql_benchmarks/clickbench/init/load-${CLICKBENCH_TYPE:-single}.sql
+
+assert I
+SELECT COUNT(*) > 0 from hits;
+----
+true
+
+run
+SELECT MAX(c) FROM (
+ SELECT covar_samp("ResolutionWidth", "ResolutionHeight") as c
+ FROM hits
+ GROUP BY "UserID"
+);
+
+result sql_benchmarks/clickbench_extended/results/q15.csv
diff --git a/benchmarks/sql_benchmarks/clickbench_extended/benchmarks/q16.benchmark b/benchmarks/sql_benchmarks/clickbench_extended/benchmarks/q16.benchmark
new file mode 100644
index 0000000000..3d6daf455f
--- /dev/null
+++ b/benchmarks/sql_benchmarks/clickbench_extended/benchmarks/q16.benchmark
@@ -0,0 +1,21 @@
+name Q16
+group clickbench_extended
+subgroup ${CLICKBENCH_TYPE:-single}
+
+init sql_benchmarks/clickbench/init/set_config.sql
+
+load sql_benchmarks/clickbench/init/load-${CLICKBENCH_TYPE:-single}.sql
+
+assert I
+SELECT COUNT(*) > 0 from hits;
+----
+true
+
+run
+SELECT MAX(c) FROM (
+ SELECT covar_samp("ResolutionWidth", "ResolutionHeight") as c
+ FROM hits
+ GROUP BY "RegionID"
+);
+
+result sql_benchmarks/clickbench_extended/results/q16.csv
diff --git a/benchmarks/tests/clickbench_parity.rs b/benchmarks/tests/clickbench_parity.rs
new file mode 100644
index 0000000000..98063933b7
--- /dev/null
+++ b/benchmarks/tests/clickbench_parity.rs
@@ -0,0 +1,448 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::fs;
+use std::path::{Path, PathBuf};
+
+const ROOT: &str = env!("CARGO_MANIFEST_DIR");
+
+struct IntentionalDifference {
+ suite: &'static str,
+ id: u32,
+ reason: &'static str,
+}
+
+const INTENTIONAL_DIFFERENCES: &[IntentionalDifference] = &[];
+
+#[test]
+fn clickbench_queries_stay_in_sync() {
+ check_suite("clickbench", "queries", "clickbench.slt");
+ check_suite("clickbench_extended", "extended", "clickbench_extended.slt");
+}
+
+fn check_suite(suite: &str, legacy_directory: &str, slt_file: &str) {
+ let benchmark_directory = Path::new(ROOT)
+ .join("sql_benchmarks")
+ .join(suite)
+ .join("benchmarks");
+ let benchmarks = read_benchmarks(&benchmark_directory);
+ let slt_path = Path::new(ROOT)
+ .join("../datafusion/sqllogictest/test_files")
+ .join(slt_file);
+ let slt_queries = read_slt_queries(&slt_path);
+
+ for (id, benchmark) in &benchmarks {
+ if is_intentional_difference(suite, *id) {
+ continue;
+ }
+ assert!(
+ slt_queries.contains(&benchmark.sql),
+ "ClickBench query {id} differs between {} and {}",
+ benchmark.path.display(),
+ slt_path.display(),
+ );
+ }
+
+ let legacy_directory = Path::new(ROOT)
+ .join("queries/clickbench")
+ .join(legacy_directory);
+ let legacy_queries = read_legacy_queries(&legacy_directory);
+ assert_same_ids(
+ suite,
+ &benchmark_directory,
+ &benchmarks,
+ &legacy_directory,
+ &legacy_queries,
+ );
+
+ for (id, benchmark) in benchmarks {
+ if is_intentional_difference(suite, id) {
+ continue;
+ }
+ let legacy = &legacy_queries[&id];
+ assert_eq!(
+ benchmark.sql,
+ legacy.sql,
+ "ClickBench query {id} differs between {} and {}",
+ benchmark.path.display(),
+ legacy.path.display(),
+ );
+ }
+}
+
+fn is_intentional_difference(suite: &str, id: u32) -> bool {
+ INTENTIONAL_DIFFERENCES.iter().any(|difference| {
+ difference.suite == suite && difference.id == id && !difference.reason.is_empty()
+ })
+}
+
+fn assert_same_ids(
+ suite: &str,
+ benchmark_directory: &Path,
+ benchmarks: &BTreeMap<u32, Query>,
+ legacy_directory: &Path,
+ legacy_queries: &BTreeMap<u32, Query>,
+) {
+ let benchmark_ids = benchmarks.keys().copied().collect::<BTreeSet<_>>();
+ let legacy_ids = legacy_queries.keys().copied().collect::<BTreeSet<_>>();
+ let missing_benchmarks = legacy_ids
+ .difference(&benchmark_ids)
+ .map(|id| legacy_queries[id].path.display().to_string())
+ .collect::<Vec<_>>();
+ let missing_legacy = benchmark_ids
+ .difference(&legacy_ids)
+ .map(|id| benchmarks[id].path.display().to_string())
+ .collect::<Vec<_>>();
+
+ assert!(
+ missing_benchmarks.is_empty() && missing_legacy.is_empty(),
+ "ClickBench {suite} query IDs differ between {} and {}. Missing benchmarks: {missing_benchmarks:?}. Missing legacy queries: {missing_legacy:?}",
+ benchmark_directory.display(),
+ legacy_directory.display(),
+ );
+}
+
+struct Query {
+ path: PathBuf,
+ sql: String,
+}
+
+fn read_benchmarks(directory: &Path) -> BTreeMap<u32, Query> {
+ read_queries(directory, "benchmark", |contents| {
+ let (_, sql) = contents
+ .split_once("\nrun\n")
+ .expect("benchmark must contain a run block");
+ sql.split_once("\nresult ")
+ .expect("benchmark run block must be followed by result")
+ .0
+ })
+}
+
+fn read_legacy_queries(directory: &Path) -> BTreeMap<u32, Query> {
+ read_queries(directory, "sql", |contents| contents)
+}
+
+fn read_queries(
+ directory: &Path,
+ extension: &str,
+ extract_sql: impl Fn(&str) -> &str,
+) -> BTreeMap<u32, Query> {
+ let mut queries = BTreeMap::new();
+ for entry in fs::read_dir(directory).unwrap() {
+ let path = entry.unwrap().path();
+ if path.extension().and_then(|extension| extension.to_str()) != Some(extension) {
+ continue;
+ }
+
+ let stem = path.file_stem().unwrap().to_str().unwrap();
+ let id = stem.strip_prefix('q').unwrap().parse().unwrap();
+ let contents = fs::read_to_string(&path).unwrap();
+ let old = queries.insert(
+ id,
+ Query {
+ path,
+ sql: normalize_sql(extract_sql(&contents)),
+ },
+ );
+ assert!(old.is_none(), "duplicate ClickBench query ID {id}");
+ }
+ queries
+}
+
+fn read_slt_queries(path: &Path) -> BTreeSet<String> {
+ let contents = fs::read_to_string(path).unwrap();
+ let mut queries = BTreeSet::new();
+ let mut offset = 0;
+
+ while let Some(directive_offset) = find_query_directive(&contents[offset..]) {
+ let directive = offset + directive_offset;
+ let sql_offset = contents[directive..]
+ .find('\n')
+ .map(|offset| directive + offset + 1)
+ .expect("query directive must be followed by SQL");
+ let boundary_offset = find_slt_boundary(&contents[sql_offset..])
+ .expect("query SQL must be followed by expected output");
+ queries.insert(normalize_sql(
+ &contents[sql_offset..sql_offset + boundary_offset],
+ ));
+ offset = sql_offset + boundary_offset + "\n----".len();
+ offset += skip_expected_output(&contents[offset..]);
+ }
+
+ queries
+}
+
+fn skip_expected_output(contents: &str) -> usize {
+ contents
+ .find("\n\n")
+ .map(|offset| offset + 2)
+ .unwrap_or(contents.len())
+}
+
+fn find_query_directive(contents: &str) -> Option<usize> {
+ contents
+ .strip_prefix("query ")
+ .map(|_| 0)
+ .or_else(|| contents.find("\nquery ").map(|offset| offset + 1))
+}
+
+fn find_slt_boundary(sql: &str) -> Option<usize> {
+ let mut index = 0;
+
+ while index < sql.len() {
+ let remaining = &sql[index..];
+ let character = remaining.chars().next().unwrap();
+
+ if remaining.starts_with("\n----") {
+ return Some(index);
+ }
+ if matches!(character, '\'' | '"') {
+ index += quoted_end(
+ remaining,
+ character,
+ character == '\'' && is_escaped_string_start(sql, index),
+ );
+ } else if let Some(delimiter) = dollar_quote_delimiter(remaining) {
+ let content = &remaining[delimiter.len()..];
+ index += content
+ .find(delimiter)
+ .map(|end| delimiter.len() + end + delimiter.len())
+ .unwrap_or(remaining.len());
+ } else {
+ index += character.len_utf8();
+ }
+ }
+
+ None
+}
+
+fn normalize_sql(sql: &str) -> String {
+ let mut normalized = String::with_capacity(sql.len());
+ let mut pending_whitespace = false;
+ let mut index = 0;
+
+ while index < sql.len() {
+ let remaining = &sql[index..];
+ let character = remaining.chars().next().unwrap();
+
+ if character.is_whitespace() {
+ pending_whitespace = true;
+ index += character.len_utf8();
+ } else if remaining.starts_with("--") {
+ index += remaining.find('\n').unwrap_or(remaining.len());
+ pending_whitespace = true;
+ } else if matches!(character, '\'' | '"') {
+ push_pending_whitespace(&mut normalized, &mut pending_whitespace);
+ let end = quoted_end(
+ remaining,
+ character,
+ character == '\'' && is_escaped_string_start(sql, index),
+ );
+ normalized.push_str(&remaining[..end]);
+ index += end;
+ } else if let Some(delimiter) = dollar_quote_delimiter(remaining) {
+ push_pending_whitespace(&mut normalized, &mut pending_whitespace);
+ let content = &remaining[delimiter.len()..];
+ let end = content
+ .find(delimiter)
+ .map(|end| delimiter.len() + end + delimiter.len())
+ .unwrap_or(remaining.len());
+ normalized.push_str(&remaining[..end]);
+ index += end;
+ } else {
+ push_pending_whitespace(&mut normalized, &mut pending_whitespace);
+ normalized.push(character);
+ index += character.len_utf8();
+ }
+ }
+
+ normalized
+}
+
+fn push_pending_whitespace(output: &mut String, pending_whitespace: &mut bool) {
+ if *pending_whitespace && !output.is_empty() {
+ output.push(' ');
+ }
+ *pending_whitespace = false;
+}
+
+fn is_escaped_string_start(sql: &str, quote_index: usize) -> bool {
+ let Some(prefix) = quote_index
+ .checked_sub(1)
+ .and_then(|index| sql.as_bytes().get(index))
+ else {
+ return false;
+ };
+
+ matches!(prefix, b'e' | b'E')
+ && quote_index
+ .checked_sub(2)
+ .and_then(|index| sql.as_bytes().get(index))
+ .is_none_or(|character| {
+ !character.is_ascii_alphanumeric() && *character != b'_'
+ })
+}
+
+fn quoted_end(sql: &str, quote: char, backslash_escapes: bool) -> usize {
+ let mut index = quote.len_utf8();
+
+ while index < sql.len() {
+ let remaining = &sql[index..];
+ let character = remaining.chars().next().unwrap();
+ index += character.len_utf8();
+
+ if character == '\\' && backslash_escapes {
+ if let Some(escaped) = sql[index..].chars().next() {
+ index += escaped.len_utf8();
+ }
+ } else if character == quote {
+ if remaining[character.len_utf8()..].starts_with(quote) {
+ index += quote.len_utf8();
+ } else {
+ return index;
+ }
+ }
+ }
+
+ sql.len()
+}
+
+fn dollar_quote_delimiter(sql: &str) -> Option<&str> {
+ let mut characters = sql.char_indices();
+ if characters.next()?.1 != '$' {
+ return None;
+ }
+
+ let (first_index, first) = characters.next()?;
+ if first == '$' {
+ return Some(&sql[..first_index + first.len_utf8()]);
+ }
+ if !matches!(first, 'a'..='z' | 'A'..='Z' | '_') {
+ return None;
+ }
+
+ for (index, character) in characters {
+ if character == '$' {
+ return Some(&sql[..index + character.len_utf8()]);
+ }
+ if !matches!(character, 'a'..='z' | 'A'..='Z' | '0'..='9' | '_') {
+ return None;
+ }
+ }
+
+ None
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::BTreeSet;
+ use std::fs;
+
+ use super::{normalize_sql, read_slt_queries};
+
+ #[test]
+ fn reads_interleaved_and_multiline_slt_queries() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("queries.slt");
+ fs::write(
+ &path,
+ "query I\nSELECT unrelated;\n----\n1\n\nquery I\nSELECT\n \"ClickBenchColumn\"\nFROM hits;\n----\n1\n",
+ )
+ .unwrap();
+
+ assert_eq!(
+ read_slt_queries(&path),
+ BTreeSet::from([
+ "SELECT unrelated;".to_string(),
+ "SELECT \"ClickBenchColumn\" FROM hits;".to_string(),
+ ])
+ );
+ }
+
+ #[test]
+ fn ignores_slt_directives_and_boundaries_in_string_literals() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("queries.slt");
+ fs::write(
+ &path,
+ "query T\nSELECT 'first line\nquery I\n----\nlast line';\n----\nfirst line\\nquery I\\n----\\nlast line\n",
+ )
+ .unwrap();
+
+ assert_eq!(
+ read_slt_queries(&path),
+ BTreeSet::from(
+ ["SELECT 'first line\nquery I\n----\nlast line';".to_string()]
+ )
+ );
+ }
+
+ #[test]
+ fn ignores_query_directives_in_expected_output() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("queries.slt");
+ fs::write(
+ &path,
+ "query I\nSELECT 1;\n----\nquery I\n\nquery I\nSELECT 2;\n----\n2\n",
+ )
+ .unwrap();
+
+ assert_eq!(
+ read_slt_queries(&path),
+ BTreeSet::from(["SELECT 1;".to_string(), "SELECT 2;".to_string()])
+ );
+ }
+
+ #[test]
+ fn normalizes_comments_and_whitespace_only() {
+ assert_eq!(
+ normalize_sql("-- comment\nSELECT a -- comment\nFROM\tb;"),
+ "SELECT a FROM b;"
+ );
+ }
+
+ #[test]
+ fn preserves_comment_markers_in_string_literals() {
+ assert_eq!(
+ normalize_sql("SELECT 'it''s -- a value' FROM \"a\"\"-- table\"; -- comment"),
+ "SELECT 'it''s -- a value' FROM \"a\"\"-- table\";"
+ );
+ }
+
+ #[test]
+ fn preserves_quoted_content() {
+ assert_ne!(
+ normalize_sql("SELECT 'a b';"),
+ normalize_sql("SELECT 'a b';")
+ );
+ assert_ne!(
+ normalize_sql("SELECT \"a b\";"),
+ normalize_sql("SELECT \"a b\";")
+ );
+ assert_ne!(
+ normalize_sql("SELECT 'a\nb';"),
+ normalize_sql("SELECT 'a b';")
+ );
+ assert_eq!(
+ normalize_sql(
+ "SELECT E'a\\' b -- value', $tag$c d -- value$tag$; -- comment"
+ ),
+ "SELECT E'a\\' b -- value', $tag$c d -- value$tag$;"
+ );
+ }
+}
diff --git a/datafusion/sqllogictest/test_files/clickbench.slt b/datafusion/sqllogictest/test_files/clickbench.slt
index 07cb828d65..76c6f6036a 100644
--- a/datafusion/sqllogictest/test_files/clickbench.slt
+++ b/datafusion/sqllogictest/test_files/clickbench.slt
@@ -53,6 +53,11 @@ SELECT "EventDate" FROM hits_raw LIMIT 1;
# queries.sql came from
# https://github.com/ClickHouse/ClickBench/blob/8b9e3aa05ea18afa427f14909ddc678b8ef0d5e6/datafusion/queries.sql
+#
+# Keep benchmark query bodies synchronized with:
+# - benchmarks/queries/clickbench/queries/qN.sql
+# - benchmarks/sql_benchmarks/clickbench/benchmarks/qNN.benchmark
+# The `clickbench_parity` test enforces this contract.
## Q0
query TT
diff --git a/datafusion/sqllogictest/test_files/clickbench_extended.slt b/datafusion/sqllogictest/test_files/clickbench_extended.slt
index 203d0aa5f2..b41d8b4dea 100644
--- a/datafusion/sqllogictest/test_files/clickbench_extended.slt
+++ b/datafusion/sqllogictest/test_files/clickbench_extended.slt
@@ -23,8 +23,10 @@ CREATE EXTERNAL TABLE hits
STORED AS PARQUET
LOCATION '../core/tests/data/clickbench_hits_10.parquet';
-# If you change any of these queries, please change the corresponding query in
-# benchmarks/queries/clickbench/extended.sql and update the README.
+# Keep query bodies synchronized with:
+# - benchmarks/queries/clickbench/extended/qN.sql
+# - benchmarks/sql_benchmarks/clickbench_extended/benchmarks/qNN.benchmark
+# The `clickbench_parity` test enforces this contract.
query III
SELECT COUNT(DISTINCT "SearchPhrase"), COUNT(DISTINCT "MobilePhone"), COUNT(DISTINCT "MobilePhoneModel") FROM hits;
@@ -130,6 +132,23 @@ query TII
SELECT "SearchPhrase", COUNT(*) AS c, COUNT(DISTINCT "MobilePhoneModel") AS models FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10;
----
+query R
+SELECT MAX(c) FROM (
+ SELECT covar_samp("ResolutionWidth", "ResolutionHeight") as c
+ FROM hits
+ GROUP BY "UserID"
+);
+----
+0
+
+query R
+SELECT MAX(c) FROM (
+ SELECT covar_samp("ResolutionWidth", "ResolutionHeight") as c
+ FROM hits
+ GROUP BY "RegionID"
+);
+----
+0
statement ok
drop table hits;