Apache DataFusion SQL Query Engine
SQL benchmark configuration parity updates - #24778 2 (#24803)
Bruce Ritchie83022b192e1a
diff --git a/benchmarks/sql_benchmarks/sort_tpch/init/load.sql b/benchmarks/sql_benchmarks/sort_tpch/init/load.sqlindex 395d8da009..71d158be9b 100644--- a/benchmarks/sql_benchmarks/sort_tpch/init/load.sql+++ b/benchmarks/sql_benchmarks/sort_tpch/init/load.sql@@ -1,3 +1,4 @@-CREATE EXTERNAL TABLE lineitem_raw STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/lineitem/lineitem.1.parquet';--CREATE TABLE lineitem as (SELECT * FROM lineitem_raw${BENCH_SORTED:-false| order by l_orderkey asc| });\ No newline at end of file+CREATE EXTERNAL TABLE lineitem+STORED AS PARQUET+LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/lineitem'+${BENCH_SORTED:-false|WITH ORDER (l_orderkey ASC NULLS LAST)| };diff --git a/benchmarks/sql_benchmarks/tpcds/init/load.sql b/benchmarks/sql_benchmarks/tpcds/init/load.sqlindex 6b89199646..a5d572be60 100644--- a/benchmarks/sql_benchmarks/tpcds/init/load.sql+++ b/benchmarks/sql_benchmarks/tpcds/init/load.sql@@ -1,47 +1,545 @@-CREATE EXTERNAL TABLE call_center STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/call_center.parquet';+-- Full schema's are required to define primary keys (to match the rust native version)+CREATE EXTERNAL TABLE call_center+(+ cc_call_center_sk INT,+ cc_call_center_id VARCHAR,+ cc_rec_start_date DATE,+ cc_rec_end_date DATE,+ cc_closed_date_sk INT,+ cc_open_date_sk INT,+ cc_name VARCHAR,+ cc_class VARCHAR,+ cc_employees INT,+ cc_sq_ft INT,+ cc_hours VARCHAR,+ cc_manager VARCHAR,+ cc_mkt_id INT,+ cc_mkt_class VARCHAR,+ cc_mkt_desc VARCHAR,+ cc_market_manager VARCHAR,+ cc_division INT,+ cc_division_name VARCHAR,+ cc_company INT,+ cc_company_name VARCHAR,+ cc_street_number VARCHAR,+ cc_street_name VARCHAR,+ cc_street_type VARCHAR,+ cc_suite_number VARCHAR,+ cc_city VARCHAR,+ cc_county VARCHAR,+ cc_state VARCHAR,+ cc_zip VARCHAR,+ cc_country VARCHAR,+ cc_gmt_offset DECIMAL(5, 2),+ cc_tax_percentage DECIMAL(5, 2),+ PRIMARY KEY (cc_call_center_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/call_center.parquet';-CREATE EXTERNAL TABLE catalog_page STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/catalog_page.parquet';+CREATE EXTERNAL TABLE catalog_page+(+ cp_catalog_page_sk INT,+ cp_catalog_page_id VARCHAR,+ cp_start_date_sk INT,+ cp_end_date_sk INT,+ cp_department VARCHAR,+ cp_catalog_number INT,+ cp_catalog_page_number INT,+ cp_description VARCHAR,+ cp_type VARCHAR,+ PRIMARY KEY (cp_catalog_page_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/catalog_page.parquet';-CREATE EXTERNAL TABLE catalog_returns STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/catalog_returns.parquet';+CREATE EXTERNAL TABLE catalog_returns+(+ cr_returned_date_sk INT,+ cr_returned_time_sk INT,+ cr_item_sk INT,+ cr_refunded_customer_sk INT,+ cr_refunded_cdemo_sk INT,+ cr_refunded_hdemo_sk INT,+ cr_refunded_addr_sk INT,+ cr_returning_customer_sk INT,+ cr_returning_cdemo_sk INT,+ cr_returning_hdemo_sk INT,+ cr_returning_addr_sk INT,+ cr_call_center_sk INT,+ cr_catalog_page_sk INT,+ cr_ship_mode_sk INT,+ cr_warehouse_sk INT,+ cr_reason_sk INT,+ cr_order_number INT,+ cr_return_quantity INT,+ cr_return_amount DECIMAL(7, 2),+ cr_return_tax DECIMAL(7, 2),+ cr_return_amt_inc_tax DECIMAL(7, 2),+ cr_fee DECIMAL(7, 2),+ cr_return_ship_cost DECIMAL(7, 2),+ cr_refunded_cash DECIMAL(7, 2),+ cr_reversed_charge DECIMAL(7, 2),+ cr_store_credit DECIMAL(7, 2),+ cr_net_loss DECIMAL(7, 2),+ PRIMARY KEY (cr_item_sk, cr_order_number)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/catalog_returns.parquet';-CREATE EXTERNAL TABLE catalog_sales STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/catalog_sales.parquet';+CREATE EXTERNAL TABLE catalog_sales+(+ cs_sold_date_sk INT,+ cs_sold_time_sk INT,+ cs_ship_date_sk INT,+ cs_bill_customer_sk INT,+ cs_bill_cdemo_sk INT,+ cs_bill_hdemo_sk INT,+ cs_bill_addr_sk INT,+ cs_ship_customer_sk INT,+ cs_ship_cdemo_sk INT,+ cs_ship_hdemo_sk INT,+ cs_ship_addr_sk INT,+ cs_call_center_sk INT,+ cs_catalog_page_sk INT,+ cs_ship_mode_sk INT,+ cs_warehouse_sk INT,+ cs_item_sk INT,+ cs_promo_sk INT,+ cs_order_number INT,+ cs_quantity INT,+ cs_wholesale_cost DECIMAL(7, 2),+ cs_list_price DECIMAL(7, 2),+ cs_sales_price DECIMAL(7, 2),+ cs_ext_discount_amt DECIMAL(7, 2),+ cs_ext_sales_price DECIMAL(7, 2),+ cs_ext_wholesale_cost DECIMAL(7, 2),+ cs_ext_list_price DECIMAL(7, 2),+ cs_ext_tax DECIMAL(7, 2),+ cs_coupon_amt DECIMAL(7, 2),+ cs_ext_ship_cost DECIMAL(7, 2),+ cs_net_paid DECIMAL(7, 2),+ cs_net_paid_inc_tax DECIMAL(7, 2),+ cs_net_paid_inc_ship DECIMAL(7, 2),+ cs_net_paid_inc_ship_tax DECIMAL(7, 2),+ cs_net_profit DECIMAL(7, 2),+ PRIMARY KEY (cs_item_sk, cs_order_number)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/catalog_sales.parquet';-CREATE EXTERNAL TABLE customer STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/customer.parquet';+CREATE EXTERNAL TABLE customer+(+ c_customer_sk INT,+ c_customer_id VARCHAR,+ c_current_cdemo_sk INT,+ c_current_hdemo_sk INT,+ c_current_addr_sk INT,+ c_first_shipto_date_sk INT,+ c_first_sales_date_sk INT,+ c_salutation VARCHAR,+ c_first_name VARCHAR,+ c_last_name VARCHAR,+ c_preferred_cust_flag VARCHAR,+ c_birth_day INT,+ c_birth_month INT,+ c_birth_year INT,+ c_birth_country VARCHAR,+ c_login VARCHAR,+ c_email_address VARCHAR,+ c_last_review_date_sk VARCHAR,+ PRIMARY KEY (c_customer_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/customer.parquet';-CREATE EXTERNAL TABLE customer_address STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/customer_address.parquet';+CREATE EXTERNAL TABLE customer_address+(+ ca_address_sk INT,+ ca_address_id VARCHAR,+ ca_street_number VARCHAR,+ ca_street_name VARCHAR,+ ca_street_type VARCHAR,+ ca_suite_number VARCHAR,+ ca_city VARCHAR,+ ca_county VARCHAR,+ ca_state VARCHAR,+ ca_zip VARCHAR,+ ca_country VARCHAR,+ ca_gmt_offset DECIMAL(5, 2),+ ca_location_type VARCHAR,+ PRIMARY KEY (ca_address_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/customer_address.parquet';-CREATE EXTERNAL TABLE customer_demographics STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/customer_demographics.parquet';+CREATE EXTERNAL TABLE customer_demographics+(+ cd_demo_sk INT,+ cd_gender VARCHAR,+ cd_marital_status VARCHAR,+ cd_education_status VARCHAR,+ cd_purchase_estimate INT,+ cd_credit_rating VARCHAR,+ cd_dep_count INT,+ cd_dep_employed_count INT,+ cd_dep_college_count INT,+ PRIMARY KEY (cd_demo_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/customer_demographics.parquet';-CREATE EXTERNAL TABLE date_dim STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/date_dim.parquet';+CREATE EXTERNAL TABLE date_dim+(+ d_date_sk INT,+ d_date_id VARCHAR,+ d_date DATE,+ d_month_seq INT,+ d_week_seq INT,+ d_quarter_seq INT,+ d_year INT,+ d_dow INT,+ d_moy INT,+ d_dom INT,+ d_qoy INT,+ d_fy_year INT,+ d_fy_quarter_seq INT,+ d_fy_week_seq INT,+ d_day_name VARCHAR,+ d_quarter_name VARCHAR,+ d_holiday VARCHAR,+ d_weekend VARCHAR,+ d_following_holiday VARCHAR,+ d_first_dom INT,+ d_last_dom INT,+ d_same_day_ly INT,+ d_same_day_lq INT,+ d_current_day VARCHAR,+ d_current_week VARCHAR,+ d_current_month VARCHAR,+ d_current_quarter VARCHAR,+ d_current_year VARCHAR,+ PRIMARY KEY (d_date_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/date_dim.parquet';-CREATE EXTERNAL TABLE household_demographics STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/household_demographics.parquet';+CREATE EXTERNAL TABLE household_demographics+(+ hd_demo_sk INT,+ hd_income_band_sk INT,+ hd_buy_potential VARCHAR,+ hd_dep_count INT,+ hd_vehicle_count INT,+ PRIMARY KEY (hd_demo_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/household_demographics.parquet';-CREATE EXTERNAL TABLE income_band STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/income_band.parquet';+CREATE EXTERNAL TABLE income_band+(+ ib_income_band_sk INT,+ ib_lower_bound INT,+ ib_upper_bound INT,+ PRIMARY KEY (ib_income_band_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/income_band.parquet';-CREATE EXTERNAL TABLE inventory STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/inventory.parquet';+CREATE EXTERNAL TABLE inventory+(+ inv_date_sk INT,+ inv_item_sk INT,+ inv_warehouse_sk INT,+ inv_quantity_on_hand INT,+ PRIMARY KEY (inv_date_sk, inv_item_sk, inv_warehouse_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/inventory.parquet';-CREATE EXTERNAL TABLE item STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/item.parquet';+CREATE EXTERNAL TABLE item+(+ i_item_sk INT,+ i_item_id VARCHAR,+ i_rec_start_date DATE,+ i_rec_end_date DATE,+ i_item_desc VARCHAR,+ i_current_price DECIMAL(7, 2),+ i_wholesale_cost DECIMAL(7, 2),+ i_brand_id INT,+ i_brand VARCHAR,+ i_class_id INT,+ i_class VARCHAR,+ i_category_id INT,+ i_category VARCHAR,+ i_manufact_id INT,+ i_manufact VARCHAR,+ i_size VARCHAR,+ i_formulation VARCHAR,+ i_color VARCHAR,+ i_units VARCHAR,+ i_container VARCHAR,+ i_manager_id INT,+ i_product_name VARCHAR,+ PRIMARY KEY (i_item_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/item.parquet';-CREATE EXTERNAL TABLE promotion STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/promotion.parquet';+CREATE EXTERNAL TABLE promotion+(+ p_promo_sk INT,+ p_promo_id VARCHAR,+ p_start_date_sk INT,+ p_end_date_sk INT,+ p_item_sk INT,+ p_cost DECIMAL(15, 2),+ p_response_target INT,+ p_promo_name VARCHAR,+ p_channel_dmail VARCHAR,+ p_channel_email VARCHAR,+ p_channel_catalog VARCHAR,+ p_channel_tv VARCHAR,+ p_channel_radio VARCHAR,+ p_channel_press VARCHAR,+ p_channel_event VARCHAR,+ p_channel_demo VARCHAR,+ p_channel_details VARCHAR,+ p_purpose VARCHAR,+ p_discount_active VARCHAR,+ PRIMARY KEY (p_promo_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/promotion.parquet';-CREATE EXTERNAL TABLE reason STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/reason.parquet';+CREATE EXTERNAL TABLE reason+(+ r_reason_sk INT,+ r_reason_id VARCHAR,+ r_reason_desc VARCHAR,+ PRIMARY KEY (r_reason_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/reason.parquet';-CREATE EXTERNAL TABLE ship_mode STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/ship_mode.parquet';+CREATE EXTERNAL TABLE ship_mode+(+ sm_ship_mode_sk INT,+ sm_ship_mode_id VARCHAR,+ sm_type VARCHAR,+ sm_code VARCHAR,+ sm_carrier VARCHAR,+ sm_contract VARCHAR,+ PRIMARY KEY (sm_ship_mode_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/ship_mode.parquet';-CREATE EXTERNAL TABLE store STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/store.parquet';+CREATE EXTERNAL TABLE store+(+ s_store_sk INT,+ s_store_id VARCHAR,+ s_rec_start_date DATE,+ s_rec_end_date DATE,+ s_closed_date_sk INT,+ s_store_name VARCHAR,+ s_number_employees INT,+ s_floor_space INT,+ s_hours VARCHAR,+ s_manager VARCHAR,+ s_market_id INT,+ s_geography_class VARCHAR,+ s_market_desc VARCHAR,+ s_market_manager VARCHAR,+ s_division_id INT,+ s_division_name VARCHAR,+ s_company_id INT,+ s_company_name VARCHAR,+ s_street_number VARCHAR,+ s_street_name VARCHAR,+ s_street_type VARCHAR,+ s_suite_number VARCHAR,+ s_city VARCHAR,+ s_county VARCHAR,+ s_state VARCHAR,+ s_zip VARCHAR,+ s_country VARCHAR,+ s_gmt_offset DECIMAL(5, 2),+ s_tax_precentage DECIMAL(5, 2),+ PRIMARY KEY (s_store_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/store.parquet';-CREATE EXTERNAL TABLE store_returns STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/store_returns.parquet';+CREATE EXTERNAL TABLE store_returns+(+ sr_returned_date_sk INT,+ sr_return_time_sk INT,+ sr_item_sk INT,+ sr_customer_sk INT,+ sr_cdemo_sk INT,+ sr_hdemo_sk INT,+ sr_addr_sk INT,+ sr_store_sk INT,+ sr_reason_sk INT,+ sr_ticket_number INT,+ sr_return_quantity INT,+ sr_return_amt DECIMAL(7, 2),+ sr_return_tax DECIMAL(7, 2),+ sr_return_amt_inc_tax DECIMAL(7, 2),+ sr_fee DECIMAL(7, 2),+ sr_return_ship_cost DECIMAL(7, 2),+ sr_refunded_cash DECIMAL(7, 2),+ sr_reversed_charge DECIMAL(7, 2),+ sr_store_credit DECIMAL(7, 2),+ sr_net_loss DECIMAL(7, 2),+ PRIMARY KEY (sr_item_sk, sr_ticket_number)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/store_returns.parquet';-CREATE EXTERNAL TABLE store_sales STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/store_sales.parquet';+CREATE EXTERNAL TABLE store_sales+(+ ss_sold_date_sk INT,+ ss_sold_time_sk INT,+ ss_item_sk INT,+ ss_customer_sk INT,+ ss_cdemo_sk INT,+ ss_hdemo_sk INT,+ ss_addr_sk INT,+ ss_store_sk INT,+ ss_promo_sk INT,+ ss_ticket_number INT,+ ss_quantity INT,+ ss_wholesale_cost DECIMAL(7, 2),+ ss_list_price DECIMAL(7, 2),+ ss_sales_price DECIMAL(7, 2),+ ss_ext_discount_amt DECIMAL(7, 2),+ ss_ext_sales_price DECIMAL(7, 2),+ ss_ext_wholesale_cost DECIMAL(7, 2),+ ss_ext_list_price DECIMAL(7, 2),+ ss_ext_tax DECIMAL(7, 2),+ ss_coupon_amt DECIMAL(7, 2),+ ss_net_paid DECIMAL(7, 2),+ ss_net_paid_inc_tax DECIMAL(7, 2),+ ss_net_profit DECIMAL(7, 2),+ PRIMARY KEY (ss_item_sk, ss_ticket_number)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/store_sales.parquet';-CREATE EXTERNAL TABLE time_dim STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/time_dim.parquet';+CREATE EXTERNAL TABLE time_dim+(+ t_time_sk INT,+ t_time_id VARCHAR,+ t_time INT,+ t_hour INT,+ t_minute INT,+ t_second INT,+ t_am_pm VARCHAR,+ t_shift VARCHAR,+ t_sub_shift VARCHAR,+ t_meal_time VARCHAR,+ PRIMARY KEY (t_time_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/time_dim.parquet';-CREATE EXTERNAL TABLE warehouse STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/warehouse.parquet';+CREATE EXTERNAL TABLE warehouse+(+ w_warehouse_sk INT,+ w_warehouse_id VARCHAR,+ w_warehouse_name VARCHAR,+ w_warehouse_sq_ft INT,+ w_street_number VARCHAR,+ w_street_name VARCHAR,+ w_street_type VARCHAR,+ w_suite_number VARCHAR,+ w_city VARCHAR,+ w_county VARCHAR,+ w_state VARCHAR,+ w_zip VARCHAR,+ w_country VARCHAR,+ w_gmt_offset DECIMAL(5, 2),+ PRIMARY KEY (w_warehouse_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/warehouse.parquet';-CREATE EXTERNAL TABLE web_page STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/web_page.parquet';+CREATE EXTERNAL TABLE web_page+(+ wp_web_page_sk INT,+ wp_web_page_id VARCHAR,+ wp_rec_start_date DATE,+ wp_rec_end_date DATE,+ wp_creation_date_sk INT,+ wp_access_date_sk INT,+ wp_autogen_flag VARCHAR,+ wp_customer_sk INT,+ wp_url VARCHAR,+ wp_type VARCHAR,+ wp_char_count INT,+ wp_link_count INT,+ wp_image_count INT,+ wp_max_ad_count INT,+ PRIMARY KEY (wp_web_page_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/web_page.parquet';-CREATE EXTERNAL TABLE web_returns STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/web_returns.parquet';+CREATE EXTERNAL TABLE web_returns+(+ wr_returned_date_sk INT,+ wr_returned_time_sk INT,+ wr_item_sk INT,+ wr_refunded_customer_sk INT,+ wr_refunded_cdemo_sk INT,+ wr_refunded_hdemo_sk INT,+ wr_refunded_addr_sk INT,+ wr_returning_customer_sk INT,+ wr_returning_cdemo_sk INT,+ wr_returning_hdemo_sk INT,+ wr_returning_addr_sk INT,+ wr_web_page_sk INT,+ wr_reason_sk INT,+ wr_order_number INT,+ wr_return_quantity INT,+ wr_return_amt DECIMAL(7, 2),+ wr_return_tax DECIMAL(7, 2),+ wr_return_amt_inc_tax DECIMAL(7, 2),+ wr_fee DECIMAL(7, 2),+ wr_return_ship_cost DECIMAL(7, 2),+ wr_refunded_cash DECIMAL(7, 2),+ wr_reversed_charge DECIMAL(7, 2),+ wr_account_credit DECIMAL(7, 2),+ wr_net_loss DECIMAL(7, 2),+ PRIMARY KEY (wr_item_sk, wr_order_number)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/web_returns.parquet';-CREATE EXTERNAL TABLE web_sales STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/web_sales.parquet';+CREATE EXTERNAL TABLE web_sales+(+ ws_sold_date_sk INT,+ ws_sold_time_sk INT,+ ws_ship_date_sk INT,+ ws_item_sk INT,+ ws_bill_customer_sk INT,+ ws_bill_cdemo_sk INT,+ ws_bill_hdemo_sk INT,+ ws_bill_addr_sk INT,+ ws_ship_customer_sk INT,+ ws_ship_cdemo_sk INT,+ ws_ship_hdemo_sk INT,+ ws_ship_addr_sk INT,+ ws_web_page_sk INT,+ ws_web_site_sk INT,+ ws_ship_mode_sk INT,+ ws_warehouse_sk INT,+ ws_promo_sk INT,+ ws_order_number INT,+ ws_quantity INT,+ ws_wholesale_cost DECIMAL(7, 2),+ ws_list_price DECIMAL(7, 2),+ ws_sales_price DECIMAL(7, 2),+ ws_ext_discount_amt DECIMAL(7, 2),+ ws_ext_sales_price DECIMAL(7, 2),+ ws_ext_wholesale_cost DECIMAL(7, 2),+ ws_ext_list_price DECIMAL(7, 2),+ ws_ext_tax DECIMAL(7, 2),+ ws_coupon_amt DECIMAL(7, 2),+ ws_ext_ship_cost DECIMAL(7, 2),+ ws_net_paid DECIMAL(7, 2),+ ws_net_paid_inc_tax DECIMAL(7, 2),+ ws_net_paid_inc_ship DECIMAL(7, 2),+ ws_net_paid_inc_ship_tax DECIMAL(7, 2),+ ws_net_profit DECIMAL(7, 2),+ PRIMARY KEY (ws_item_sk, ws_order_number)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/web_sales.parquet';-CREATE EXTERNAL TABLE web_site STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/web_site.parquet';+CREATE EXTERNAL TABLE web_site+(+ web_site_sk INT,+ web_site_id VARCHAR,+ web_rec_start_date DATE,+ web_rec_end_date DATE,+ web_name VARCHAR,+ web_open_date_sk INT,+ web_close_date_sk INT,+ web_class VARCHAR,+ web_manager VARCHAR,+ web_mkt_id INT,+ web_mkt_class VARCHAR,+ web_mkt_desc VARCHAR,+ web_market_manager VARCHAR,+ web_company_id INT,+ web_company_name VARCHAR,+ web_street_number VARCHAR,+ web_street_name VARCHAR,+ web_street_type VARCHAR,+ web_suite_number VARCHAR,+ web_city VARCHAR,+ web_county VARCHAR,+ web_state VARCHAR,+ web_zip VARCHAR,+ web_country VARCHAR,+ web_gmt_offset DECIMAL(5, 2),+ web_tax_percentage DECIMAL(5, 2),+ PRIMARY KEY (web_site_sk)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpcds_sf${BENCH_SIZE:-1}/web_site.parquet';diff --git a/benchmarks/sql_benchmarks/tpch/init/load_csv.sql b/benchmarks/sql_benchmarks/tpch/init/load_csv.sqlindex d3594a7262..85bd8ecc68 100644--- a/benchmarks/sql_benchmarks/tpch/init/load_csv.sql+++ b/benchmarks/sql_benchmarks/tpch/init/load_csv.sql@@ -1,99 +1,100 @@CREATE EXTERNAL TABLE nation(- n_nationkey INT,- n_name CHAR(25),- n_regionkey INT,- n_comment VARCHAR(152),+ n_nationkey BIGINT NOT NULL,+ n_name CHAR(25) NOT NULL,+ n_regionkey BIGINT NOT NULL,+ n_comment VARCHAR(152) NOT NULL,PRIMARY KEY (n_nationkey)) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/nation/nation.1.csv';CREATE EXTERNAL TABLE region(- r_regionkey INT,- r_name CHAR(25),- r_comment VARCHAR(152),+ r_regionkey BIGINT NOT NULL,+ r_name CHAR(25) NOT NULL,+ r_comment VARCHAR(152) NOT NULL,PRIMARY KEY (r_regionkey)) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/region/region.1.csv';CREATE EXTERNAL TABLE supplier(- s_suppkey INT,- s_name CHAR(25),- s_address VARCHAR(40),- s_nationkey INT,- s_phone CHAR(15),- s_acctbal DECIMAL(15, 2),- s_comment VARCHAR(101),+ s_suppkey BIGINT NOT NULL,+ s_name CHAR(25) NOT NULL,+ s_address VARCHAR(40) NOT NULL,+ s_nationkey BIGINT NOT NULL,+ s_phone CHAR(15) NOT NULL,+ s_acctbal DECIMAL(15, 2) NOT NULL,+ s_comment VARCHAR(101) NOT NULL,PRIMARY KEY (s_suppkey)) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/supplier/supplier.1.csv';CREATE EXTERNAL TABLE customer(- c_custkey INT,- c_name VARCHAR(25),- c_address VARCHAR(40),- c_nationkey INT,- c_phone CHAR(15),- c_acctbal DECIMAL(15, 2),- c_mktsegment CHAR(10),- c_comment VARCHAR(117),+ c_custkey BIGINT NOT NULL,+ c_name VARCHAR(25) NOT NULL,+ c_address VARCHAR(40) NOT NULL,+ c_nationkey BIGINT NOT NULL,+ c_phone CHAR(15) NOT NULL,+ c_acctbal DECIMAL(15, 2) NOT NULL,+ c_mktsegment CHAR(10) NOT NULL,+ c_comment VARCHAR(117) NOT NULL,PRIMARY KEY (c_custkey)) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/customer/customer.1.csv';CREATE EXTERNAL TABLE part(- p_partkey INT,- p_name VARCHAR(55),- p_mfgr CHAR(25),- p_brand CHAR(10),- p_type VARCHAR(25),- p_size INT,- p_container CHAR(10),- p_retailprice DECIMAL(15, 2),- p_comment VARCHAR(23),+ p_partkey BIGINT NOT NULL,+ p_name VARCHAR(55) NOT NULL,+ p_mfgr CHAR(25) NOT NULL,+ p_brand CHAR(10) NOT NULL,+ p_type VARCHAR(25) NOT NULL,+ p_size INT NOT NULL,+ p_container CHAR(10) NOT NULL,+ p_retailprice DECIMAL(15, 2) NOT NULL,+ p_comment VARCHAR(23) NOT NULL,PRIMARY KEY (p_partkey)) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/part/part.1.csv';CREATE EXTERNAL TABLE partsupp(- ps_partkey INT,- ps_suppkey INT,- ps_availqty INT,- ps_supplycost DECIMAL(15, 2),- ps_comment VARCHAR(199),- PRIMARY KEY (ps_partkey)+ ps_partkey BIGINT NOT NULL,+ ps_suppkey BIGINT NOT NULL,+ ps_availqty INT NOT NULL,+ ps_supplycost DECIMAL(15, 2) NOT NULL,+ ps_comment VARCHAR(199) NOT NULL,+ PRIMARY KEY (ps_partkey, ps_suppkey)) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/partsupp/partsupp.1.csv';CREATE EXTERNAL TABLE orders(- o_orderkey INT,- o_custkey INT,- o_orderstatus CHAR(1),- o_totalprice DECIMAL(15, 2),- o_orderdate DATE,- o_orderpriority CHAR(15),- o_clerk CHAR(15),- o_shippriority INT,- o_comment VARCHAR(79),+ o_orderkey BIGINT NOT NULL,+ o_custkey BIGINT NOT NULL,+ o_orderstatus CHAR(1) NOT NULL,+ o_totalprice DECIMAL(15, 2) NOT NULL,+ o_orderdate DATE NOT NULL,+ o_orderpriority CHAR(15) NOT NULL,+ o_clerk CHAR(15) NOT NULL,+ o_shippriority INT NOT NULL,+ o_comment VARCHAR(79) NOT NULL,PRIMARY KEY (o_orderkey)) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/orders/orders.1.csv';CREATE EXTERNAL TABLE lineitem(- l_orderkey INT,- l_partkey INT,- l_suppkey INT,- l_linenumber INT,- l_quantity DECIMAL(15, 2),- l_extendedprice DECIMAL(15, 2),- l_discount DECIMAL(15, 2),- l_tax DECIMAL(15, 2),- l_returnflag CHAR(1),- l_linestatus CHAR(1),- l_shipdate DATE,- l_commitdate DATE,- l_receiptdate DATE,- l_shipinstruct CHAR(25),- l_shipmode CHAR(10),- l_comment VARCHAR(44)+ l_orderkey BIGINT NOT NULL,+ l_partkey BIGINT NOT NULL,+ l_suppkey BIGINT NOT NULL,+ l_linenumber INT NOT NULL,+ l_quantity DECIMAL(15, 2) NOT NULL,+ l_extendedprice DECIMAL(15, 2) NOT NULL,+ l_discount DECIMAL(15, 2) NOT NULL,+ l_tax DECIMAL(15, 2) NOT NULL,+ l_returnflag CHAR(1) NOT NULL,+ l_linestatus CHAR(1) NOT NULL,+ l_shipdate DATE NOT NULL,+ l_commitdate DATE NOT NULL,+ l_receiptdate DATE NOT NULL,+ l_shipinstruct CHAR(25) NOT NULL,+ l_shipmode CHAR(10) NOT NULL,+ l_comment VARCHAR(44) NOT NULL,+ PRIMARY KEY (l_orderkey, l_linenumber)) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/lineitem/lineitem.1.csv';diff --git a/benchmarks/sql_benchmarks/tpch/init/load_mem.sql b/benchmarks/sql_benchmarks/tpch/init/load_mem.sqlindex 16e5c0b493..fc78e0e144 100644--- a/benchmarks/sql_benchmarks/tpch/init/load_mem.sql+++ b/benchmarks/sql_benchmarks/tpch/init/load_mem.sql@@ -14,18 +14,103 @@ CREATE EXTERNAL TABLE orders_raw STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tCREATE EXTERNAL TABLE lineitem_raw STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/lineitem/lineitem.1.parquet';-CREATE TABLE nation as SELECT * FROM nation_raw;+CREATE TABLE nation+(+ n_nationkey BIGINT NOT NULL,+ n_name VARCHAR NOT NULL,+ n_regionkey BIGINT NOT NULL,+ n_comment VARCHAR NOT NULL,+ PRIMARY KEY (n_nationkey)+) AS SELECT * FROM nation_raw;-CREATE TABLE region as SELECT * FROM region_raw;+CREATE TABLE region+(+ r_regionkey BIGINT NOT NULL,+ r_name VARCHAR NOT NULL,+ r_comment VARCHAR NOT NULL,+ PRIMARY KEY (r_regionkey)+) AS SELECT * FROM region_raw;-CREATE TABLE supplier as SELECT * FROM supplier_raw;+CREATE TABLE supplier+(+ s_suppkey BIGINT NOT NULL,+ s_name VARCHAR NOT NULL,+ s_address VARCHAR NOT NULL,+ s_nationkey BIGINT NOT NULL,+ s_phone VARCHAR NOT NULL,+ s_acctbal DECIMAL(15, 2) NOT NULL,+ s_comment VARCHAR NOT NULL,+ PRIMARY KEY (s_suppkey)+) AS SELECT * FROM supplier_raw;-CREATE TABLE customer as SELECT * FROM customer_raw;+CREATE TABLE customer+(+ c_custkey BIGINT NOT NULL,+ c_name VARCHAR NOT NULL,+ c_address VARCHAR NOT NULL,+ c_nationkey BIGINT NOT NULL,+ c_phone VARCHAR NOT NULL,+ c_acctbal DECIMAL(15, 2) NOT NULL,+ c_mktsegment VARCHAR NOT NULL,+ c_comment VARCHAR NOT NULL,+ PRIMARY KEY (c_custkey)+) AS SELECT * FROM customer_raw;-CREATE TABLE part as SELECT * FROM part_raw;+CREATE TABLE part+(+ p_partkey BIGINT NOT NULL,+ p_name VARCHAR NOT NULL,+ p_mfgr VARCHAR NOT NULL,+ p_brand VARCHAR NOT NULL,+ p_type VARCHAR NOT NULL,+ p_size INT NOT NULL,+ p_container VARCHAR NOT NULL,+ p_retailprice DECIMAL(15, 2) NOT NULL,+ p_comment VARCHAR NOT NULL,+ PRIMARY KEY (p_partkey)+) AS SELECT * FROM part_raw;-CREATE TABLE partsupp as SELECT * FROM partsupp_raw;+CREATE TABLE partsupp+(+ ps_partkey BIGINT NOT NULL,+ ps_suppkey BIGINT NOT NULL,+ ps_availqty INT NOT NULL,+ ps_supplycost DECIMAL(15, 2) NOT NULL,+ ps_comment VARCHAR NOT NULL,+ PRIMARY KEY (ps_partkey, ps_suppkey)+) AS SELECT * FROM partsupp_raw;-CREATE TABLE orders as SELECT * FROM orders_raw;+CREATE TABLE orders+(+ o_orderkey BIGINT NOT NULL,+ o_custkey BIGINT NOT NULL,+ o_orderstatus VARCHAR NOT NULL,+ o_totalprice DECIMAL(15, 2) NOT NULL,+ o_orderdate DATE NOT NULL,+ o_orderpriority VARCHAR NOT NULL,+ o_clerk VARCHAR NOT NULL,+ o_shippriority INT NOT NULL,+ o_comment VARCHAR NOT NULL,+ PRIMARY KEY (o_orderkey)+) AS SELECT * FROM orders_raw;-CREATE TABLE lineitem as SELECT * FROM lineitem_raw;+CREATE TABLE lineitem+(+ l_orderkey BIGINT NOT NULL,+ l_partkey BIGINT NOT NULL,+ l_suppkey BIGINT NOT NULL,+ l_linenumber INT NOT NULL,+ l_quantity DECIMAL(15, 2) NOT NULL,+ l_extendedprice DECIMAL(15, 2) NOT NULL,+ l_discount DECIMAL(15, 2) NOT NULL,+ l_tax DECIMAL(15, 2) NOT NULL,+ l_returnflag VARCHAR NOT NULL,+ l_linestatus VARCHAR NOT NULL,+ l_shipdate DATE NOT NULL,+ l_commitdate DATE NOT NULL,+ l_receiptdate DATE NOT NULL,+ l_shipinstruct VARCHAR NOT NULL,+ l_shipmode VARCHAR NOT NULL,+ l_comment VARCHAR NOT NULL,+ PRIMARY KEY (l_orderkey, l_linenumber)+) AS SELECT * FROM lineitem_raw;diff --git a/benchmarks/sql_benchmarks/tpch/init/load_parquet.sql b/benchmarks/sql_benchmarks/tpch/init/load_parquet.sqlindex 9423e38a44..44e2738636 100644--- a/benchmarks/sql_benchmarks/tpch/init/load_parquet.sql+++ b/benchmarks/sql_benchmarks/tpch/init/load_parquet.sql@@ -1,15 +1,100 @@-CREATE EXTERNAL TABLE nation STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/nation/nation.1.parquet';+CREATE EXTERNAL TABLE nation+(+ n_nationkey BIGINT NOT NULL,+ n_name VARCHAR NOT NULL,+ n_regionkey BIGINT NOT NULL,+ n_comment VARCHAR NOT NULL,+ PRIMARY KEY (n_nationkey)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/nation/nation.1.parquet';-CREATE EXTERNAL TABLE region STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/region/region.1.parquet';+CREATE EXTERNAL TABLE region+(+ r_regionkey BIGINT NOT NULL,+ r_name VARCHAR NOT NULL,+ r_comment VARCHAR NOT NULL,+ PRIMARY KEY (r_regionkey)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/region/region.1.parquet';-CREATE EXTERNAL TABLE supplier STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/supplier/supplier.1.parquet';+CREATE EXTERNAL TABLE supplier+(+ s_suppkey BIGINT NOT NULL,+ s_name VARCHAR NOT NULL,+ s_address VARCHAR NOT NULL,+ s_nationkey BIGINT NOT NULL,+ s_phone VARCHAR NOT NULL,+ s_acctbal DECIMAL(15, 2) NOT NULL,+ s_comment VARCHAR NOT NULL,+ PRIMARY KEY (s_suppkey)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/supplier/supplier.1.parquet';-CREATE EXTERNAL TABLE customer STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/customer/customer.1.parquet';+CREATE EXTERNAL TABLE customer+(+ c_custkey BIGINT NOT NULL,+ c_name VARCHAR NOT NULL,+ c_address VARCHAR NOT NULL,+ c_nationkey BIGINT NOT NULL,+ c_phone VARCHAR NOT NULL,+ c_acctbal DECIMAL(15, 2) NOT NULL,+ c_mktsegment VARCHAR NOT NULL,+ c_comment VARCHAR NOT NULL,+ PRIMARY KEY (c_custkey)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/customer/customer.1.parquet';-CREATE EXTERNAL TABLE part STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/part/part.1.parquet';+CREATE EXTERNAL TABLE part+(+ p_partkey BIGINT NOT NULL,+ p_name VARCHAR NOT NULL,+ p_mfgr VARCHAR NOT NULL,+ p_brand VARCHAR NOT NULL,+ p_type VARCHAR NOT NULL,+ p_size INT NOT NULL,+ p_container VARCHAR NOT NULL,+ p_retailprice DECIMAL(15, 2) NOT NULL,+ p_comment VARCHAR NOT NULL,+ PRIMARY KEY (p_partkey)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/part/part.1.parquet';-CREATE EXTERNAL TABLE partsupp STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/partsupp/partsupp.1.parquet';+CREATE EXTERNAL TABLE partsupp+(+ ps_partkey BIGINT NOT NULL,+ ps_suppkey BIGINT NOT NULL,+ ps_availqty INT NOT NULL,+ ps_supplycost DECIMAL(15, 2) NOT NULL,+ ps_comment VARCHAR NOT NULL,+ PRIMARY KEY (ps_partkey, ps_suppkey)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/partsupp/partsupp.1.parquet';-CREATE EXTERNAL TABLE orders STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/orders/orders.1.parquet';+CREATE EXTERNAL TABLE orders+(+ o_orderkey BIGINT NOT NULL,+ o_custkey BIGINT NOT NULL,+ o_orderstatus VARCHAR NOT NULL,+ o_totalprice DECIMAL(15, 2) NOT NULL,+ o_orderdate DATE NOT NULL,+ o_orderpriority VARCHAR NOT NULL,+ o_clerk VARCHAR NOT NULL,+ o_shippriority INT NOT NULL,+ o_comment VARCHAR NOT NULL,+ PRIMARY KEY (o_orderkey)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/orders/orders.1.parquet';-CREATE EXTERNAL TABLE lineitem STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/lineitem/lineitem.1.parquet';+CREATE EXTERNAL TABLE lineitem+(+ l_orderkey BIGINT NOT NULL,+ l_partkey BIGINT NOT NULL,+ l_suppkey BIGINT NOT NULL,+ l_linenumber INT NOT NULL,+ l_quantity DECIMAL(15, 2) NOT NULL,+ l_extendedprice DECIMAL(15, 2) NOT NULL,+ l_discount DECIMAL(15, 2) NOT NULL,+ l_tax DECIMAL(15, 2) NOT NULL,+ l_returnflag VARCHAR NOT NULL,+ l_linestatus VARCHAR NOT NULL,+ l_shipdate DATE NOT NULL,+ l_commitdate DATE NOT NULL,+ l_receiptdate DATE NOT NULL,+ l_shipinstruct VARCHAR NOT NULL,+ l_shipmode VARCHAR NOT NULL,+ l_comment VARCHAR NOT NULL,+ PRIMARY KEY (l_orderkey, l_linenumber)+) STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/lineitem/lineitem.1.parquet';