Skip to content

Apache DataFusion SQL Query Engine

Fix ClickBench EventDate handling by casting UInt16 days-since-epoch to DATE via `hits` view (#19881)

kosiew5c47d6d13f0e
diff --git a/benchmarks/src/clickbench.rs b/benchmarks/src/clickbench.rs
index a9da57b02a..c0f911c566 100644
--- a/benchmarks/src/clickbench.rs
+++ b/benchmarks/src/clickbench.rs
@@ -29,6 +29,16 @@ use datafusion::{
use datafusion_common::exec_datafusion_err;
use datafusion_common::instant::Instant;
+/// SQL to create the hits view with proper EventDate casting.
+///
+/// ClickBench stores EventDate as UInt16 (days since 1970-01-01) for
+/// storage efficiency (2 bytes vs 4-8 bytes for date types).
+/// This view transforms it to SQL DATE type for query compatibility.
+const HITS_VIEW_DDL: &str = r#"CREATE VIEW hits AS
+SELECT * EXCEPT ("EventDate"),
+ CAST(CAST("EventDate" AS INTEGER) AS DATE) AS "EventDate"
+FROM hits_raw"#;
+
/// Driver program to run the ClickBench benchmark
///
/// The ClickBench[1] benchmarks are widely cited in the industry and
@@ -295,7 +305,7 @@ impl RunOpt {
// Build CREATE EXTERNAL TABLE DDL with WITH ORDER clause
// Schema will be automatically inferred from the Parquet file
let create_table_sql = format!(
- "CREATE EXTERNAL TABLE hits \
+ "CREATE EXTERNAL TABLE hits_raw \
STORED AS PARQUET \
LOCATION '{}' \
WITH ORDER ({} {})",
@@ -308,20 +318,34 @@ impl RunOpt {
// Execute the CREATE EXTERNAL TABLE statement
ctx.sql(&create_table_sql).await?.collect().await?;
-
- Ok(())
} else {
// Original registration without sort order
let options = Default::default();
- ctx.register_parquet("hits", path, options)
+ ctx.register_parquet("hits_raw", path, options)
.await
.map_err(|e| {
DataFusionError::Context(
- format!("Registering 'hits' as {path}"),
+ format!("Registering 'hits_raw' as {path}"),
Box::new(e),
)
- })
+ })?;
}
+
+ // Create the hits view with EventDate transformation
+ Self::create_hits_view(ctx).await
+ }
+
+ /// Creates the hits view with EventDate transformation from UInt16 to DATE.
+ ///
+ /// ClickBench encodes EventDate as UInt16 days since epoch (1970-01-01).
+ async fn create_hits_view(ctx: &SessionContext) -> Result<()> {
+ ctx.sql(HITS_VIEW_DDL).await?.collect().await.map_err(|e| {
+ DataFusionError::Context(
+ "Creating 'hits' view with EventDate transformation".to_string(),
+ Box::new(e),
+ )
+ })?;
+ Ok(())
}
fn iterations(&self) -> usize {
diff --git a/datafusion/sqllogictest/test_files/clickbench.slt b/datafusion/sqllogictest/test_files/clickbench.slt
index 4c60a4365e..42b7cfafda 100644
--- a/datafusion/sqllogictest/test_files/clickbench.slt
+++ b/datafusion/sqllogictest/test_files/clickbench.slt
@@ -26,10 +26,28 @@
# COPY (SELECT * FROM 'hits.parquet' LIMIT 10) TO 'clickbench_hits_10.parquet' (FORMAT PARQUET);
statement ok
-CREATE EXTERNAL TABLE hits
+CREATE EXTERNAL TABLE hits_raw
STORED AS PARQUET
LOCATION '../core/tests/data/clickbench_hits_10.parquet';
+# ClickBench encodes EventDate as UInt16 days since epoch.
+statement ok
+CREATE VIEW hits AS
+SELECT * EXCEPT ("EventDate"),
+ CAST(CAST("EventDate" AS INTEGER) AS DATE) AS "EventDate"
+FROM hits_raw;
+
+# Verify EventDate transformation from UInt16 to DATE
+query D
+SELECT "EventDate" FROM hits LIMIT 1;
+----
+2013-07-15
+
+# Verify the raw value is still UInt16 in hits_raw
+query I
+SELECT "EventDate" FROM hits_raw LIMIT 1;
+----
+15901
# queries.sql came from
# https://github.com/ClickHouse/ClickBench/blob/8b9e3aa05ea18afa427f14909ddc678b8ef0d5e6/datafusion/queries.sql
@@ -64,10 +82,10 @@ SELECT COUNT(DISTINCT "SearchPhrase") FROM hits;
----
1
-query II
+query DD
SELECT MIN("EventDate"), MAX("EventDate") FROM hits;
----
-15901 15901
+2013-07-15 2013-07-15
query II
SELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC;
@@ -167,7 +185,8 @@ query TTTII
SELECT "SearchPhrase", MIN("URL"), MIN("Title"), COUNT(*) AS c, COUNT(DISTINCT "UserID") FROM hits WHERE "Title" LIKE '%Google%' AND "URL" NOT LIKE '%.google.%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10;
----
-query IITIIIIIIIIIITTIIIIIIIIIITIIITIIIITTIIITIIIIIIIIIITIIIIITIIIIIITIIIIIIIIIITTTTIIIIIIIITITTITTTTTTTTTTIIII
+query IITIIIIIIIIITTIIIIIIIIIITIIITIIIITTIIITIIIIIIIIIITIIIIITIIIIIITIIIIIIIIIITTTTIIIIIIIITITTITTTTTTTTTTIIIID
+
SELECT * FROM hits WHERE "URL" LIKE '%google%' ORDER BY "EventTime" LIMIT 10;
----
@@ -262,7 +281,7 @@ query IIITTI
SELECT "TraficSourceID", "SearchEngineID", "AdvEngineID", CASE WHEN ("SearchEngineID" = 0 AND "AdvEngineID" = 0) THEN "Referer" ELSE '' END AS Src, "URL" AS Dst, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 GROUP BY "TraficSourceID", "SearchEngineID", "AdvEngineID", Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;
----
-query III
+query IDI
SELECT "URLHash", "EventDate", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 AND "TraficSourceID" IN (-1, 6) AND "RefererHash" = 3594120000172545465 GROUP BY "URLHash", "EventDate" ORDER BY PageViews DESC LIMIT 10 OFFSET 100;
----
@@ -293,4 +312,7 @@ SELECT "BrowserCountry", COUNT(DISTINCT "SocialNetwork"), COUNT(DISTINCT "HitCo
statement ok
-drop table hits;
+drop view hits;
+
+statement ok
+drop table hits_raw;