Apache DataFusion SQL Query Engine
Fix ClickBench EventDate handling by casting UInt16 days-since-epoch to DATE via `hits` view (#19881)
kosiew5c47d6d13f0e
diff --git a/benchmarks/src/clickbench.rs b/benchmarks/src/clickbench.rsindex a9da57b02a..c0f911c566 100644--- a/benchmarks/src/clickbench.rs+++ b/benchmarks/src/clickbench.rs@@ -29,6 +29,16 @@ use datafusion::{use datafusion_common::exec_datafusion_err;use datafusion_common::instant::Instant;+/// SQL to create the hits view with proper EventDate casting.+///+/// ClickBench stores EventDate as UInt16 (days since 1970-01-01) for+/// storage efficiency (2 bytes vs 4-8 bytes for date types).+/// This view transforms it to SQL DATE type for query compatibility.+const HITS_VIEW_DDL: &str = r#"CREATE VIEW hits AS+SELECT * EXCEPT ("EventDate"),+ CAST(CAST("EventDate" AS INTEGER) AS DATE) AS "EventDate"+FROM hits_raw"#;+/// Driver program to run the ClickBench benchmark////// The ClickBench[1] benchmarks are widely cited in the industry and@@ -295,7 +305,7 @@ impl RunOpt {// Build CREATE EXTERNAL TABLE DDL with WITH ORDER clause// Schema will be automatically inferred from the Parquet filelet create_table_sql = format!(- "CREATE EXTERNAL TABLE hits \+ "CREATE EXTERNAL TABLE hits_raw \STORED AS PARQUET \LOCATION '{}' \WITH ORDER ({} {})",@@ -308,20 +318,34 @@ impl RunOpt {// Execute the CREATE EXTERNAL TABLE statementctx.sql(&create_table_sql).await?.collect().await?;-- Ok(())} else {// Original registration without sort orderlet options = Default::default();- ctx.register_parquet("hits", path, options)+ ctx.register_parquet("hits_raw", path, options).await.map_err(|e| {DataFusionError::Context(- format!("Registering 'hits' as {path}"),+ format!("Registering 'hits_raw' as {path}"),Box::new(e),)- })+ })?;}++ // Create the hits view with EventDate transformation+ Self::create_hits_view(ctx).await+ }++ /// Creates the hits view with EventDate transformation from UInt16 to DATE.+ ///+ /// ClickBench encodes EventDate as UInt16 days since epoch (1970-01-01).+ async fn create_hits_view(ctx: &SessionContext) -> Result<()> {+ ctx.sql(HITS_VIEW_DDL).await?.collect().await.map_err(|e| {+ DataFusionError::Context(+ "Creating 'hits' view with EventDate transformation".to_string(),+ Box::new(e),+ )+ })?;+ Ok(())}fn iterations(&self) -> usize {diff --git a/datafusion/sqllogictest/test_files/clickbench.slt b/datafusion/sqllogictest/test_files/clickbench.sltindex 4c60a4365e..42b7cfafda 100644--- a/datafusion/sqllogictest/test_files/clickbench.slt+++ b/datafusion/sqllogictest/test_files/clickbench.slt@@ -26,10 +26,28 @@# COPY (SELECT * FROM 'hits.parquet' LIMIT 10) TO 'clickbench_hits_10.parquet' (FORMAT PARQUET);statement ok-CREATE EXTERNAL TABLE hits+CREATE EXTERNAL TABLE hits_rawSTORED AS PARQUETLOCATION '../core/tests/data/clickbench_hits_10.parquet';+# ClickBench encodes EventDate as UInt16 days since epoch.+statement ok+CREATE VIEW hits AS+SELECT * EXCEPT ("EventDate"),+ CAST(CAST("EventDate" AS INTEGER) AS DATE) AS "EventDate"+FROM hits_raw;++# Verify EventDate transformation from UInt16 to DATE+query D+SELECT "EventDate" FROM hits LIMIT 1;+----+2013-07-15++# Verify the raw value is still UInt16 in hits_raw+query I+SELECT "EventDate" FROM hits_raw LIMIT 1;+----+15901# queries.sql came from# https://github.com/ClickHouse/ClickBench/blob/8b9e3aa05ea18afa427f14909ddc678b8ef0d5e6/datafusion/queries.sql@@ -64,10 +82,10 @@ SELECT COUNT(DISTINCT "SearchPhrase") FROM hits;----1-query II+query DDSELECT MIN("EventDate"), MAX("EventDate") FROM hits;-----15901 15901+2013-07-15 2013-07-15query IISELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC;@@ -167,7 +185,8 @@ query TTTIISELECT "SearchPhrase", MIN("URL"), MIN("Title"), COUNT(*) AS c, COUNT(DISTINCT "UserID") FROM hits WHERE "Title" LIKE '%Google%' AND "URL" NOT LIKE '%.google.%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10;-----query IITIIIIIIIIIITTIIIIIIIIIITIIITIIIITTIIITIIIIIIIIIITIIIIITIIIIIITIIIIIIIIIITTTTIIIIIIIITITTITTTTTTTTTTIIII+query IITIIIIIIIIITTIIIIIIIIIITIIITIIIITTIIITIIIIIIIIIITIIIIITIIIIIITIIIIIIIIIITTTTIIIIIIIITITTITTTTTTTTTTIIIID+SELECT * FROM hits WHERE "URL" LIKE '%google%' ORDER BY "EventTime" LIMIT 10;----@@ -262,7 +281,7 @@ query IIITTISELECT "TraficSourceID", "SearchEngineID", "AdvEngineID", CASE WHEN ("SearchEngineID" = 0 AND "AdvEngineID" = 0) THEN "Referer" ELSE '' END AS Src, "URL" AS Dst, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 GROUP BY "TraficSourceID", "SearchEngineID", "AdvEngineID", Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;-----query III+query IDISELECT "URLHash", "EventDate", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 AND "TraficSourceID" IN (-1, 6) AND "RefererHash" = 3594120000172545465 GROUP BY "URLHash", "EventDate" ORDER BY PageViews DESC LIMIT 10 OFFSET 100;----@@ -293,4 +312,7 @@ SELECT "BrowserCountry", COUNT(DISTINCT "SocialNetwork"), COUNT(DISTINCT "HitCostatement ok-drop table hits;+drop view hits;++statement ok+drop table hits_raw;