From 1bca964a3de7030432b0fe8780a260d1b5c17360 Mon Sep 17 00:00:00 2001 From: Juliusz Sompolski Date: Thu, 25 May 2017 17:12:47 +0200 Subject: [PATCH] Correct types of keys --- .../spark/sql/perf/tpcds/Tables.scala | 222 +++++++++--------- 1 file changed, 111 insertions(+), 111 deletions(-) diff --git a/src/main/scala/com/databricks/spark/sql/perf/tpcds/Tables.scala b/src/main/scala/com/databricks/spark/sql/perf/tpcds/Tables.scala index 414e04d..5092c2d 100644 --- a/src/main/scala/com/databricks/spark/sql/perf/tpcds/Tables.scala +++ b/src/main/scala/com/databricks/spark/sql/perf/tpcds/Tables.scala @@ -260,25 +260,25 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend val tables = Seq( Table("catalog_sales", partitionColumns = "cs_sold_date_sk" :: Nil, - 'cs_sold_date_sk .int, - 'cs_sold_time_sk .int, - 'cs_ship_date_sk .int, - 'cs_bill_customer_sk .int, - 'cs_bill_cdemo_sk .int, - 'cs_bill_hdemo_sk .int, - 'cs_bill_addr_sk .int, - 'cs_ship_customer_sk .int, - 'cs_ship_cdemo_sk .int, - 'cs_ship_hdemo_sk .int, - 'cs_ship_addr_sk .int, - 'cs_call_center_sk .int, - 'cs_catalog_page_sk .int, - 'cs_ship_mode_sk .int, - 'cs_warehouse_sk .int, - 'cs_item_sk .int, - 'cs_promo_sk .int, - 'cs_order_number .int, - 'cs_quantity .int, + 'cs_sold_date_sk .long, + 'cs_sold_time_sk .long, + 'cs_ship_date_sk .long, + 'cs_bill_customer_sk .long, + 'cs_bill_cdemo_sk .long, + 'cs_bill_hdemo_sk .long, + 'cs_bill_addr_sk .long, + 'cs_ship_customer_sk .long, + 'cs_ship_cdemo_sk .long, + 'cs_ship_hdemo_sk .long, + 'cs_ship_addr_sk .long, + 'cs_call_center_sk .long, + 'cs_catalog_page_sk .long, + 'cs_ship_mode_sk .long, + 'cs_warehouse_sk .long, + 'cs_item_sk .long, + 'cs_promo_sk .long, + 'cs_order_number .long, + 'cs_quantity .long, 'cs_wholesale_cost .decimal(7,2), 'cs_list_price .decimal(7,2), 'cs_sales_price .decimal(7,2), @@ -296,24 +296,24 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'cs_net_profit .decimal(7,2)), Table("catalog_returns", partitionColumns = "cr_returned_date_sk" :: Nil, - 'cr_returned_date_sk .int, - 'cr_returned_time_sk .int, - 'cr_item_sk .int, - 'cr_refunded_customer_sk .int, - 'cr_refunded_cdemo_sk .int, - 'cr_refunded_hdemo_sk .int, - 'cr_refunded_addr_sk .int, - 'cr_returning_customer_sk .int, - 'cr_returning_cdemo_sk .int, - 'cr_returning_hdemo_sk .int, - 'cr_returning_addr_sk .int, - 'cr_call_center_sk .int, - 'cr_catalog_page_sk .int, - 'cr_ship_mode_sk .int, - 'cr_warehouse_sk .int, - 'cr_reason_sk .int, - 'cr_order_number .int, - 'cr_return_quantity .int, + 'cr_returned_date_sk .long, + 'cr_returned_time_sk .long, + 'cr_item_sk .long, + 'cr_refunded_customer_sk .long, + 'cr_refunded_cdemo_sk .long, + 'cr_refunded_hdemo_sk .long, + 'cr_refunded_addr_sk .long, + 'cr_returning_customer_sk .long, + 'cr_returning_cdemo_sk .long, + 'cr_returning_hdemo_sk .long, + 'cr_returning_addr_sk .long, + 'cr_call_center_sk .long, + 'cr_catalog_page_sk .long, + 'cr_ship_mode_sk .long, + 'cr_warehouse_sk .long, + 'cr_reason_sk .long, + 'cr_order_number .long, + 'cr_return_quantity .long, 'cr_return_amount .decimal(7,2), 'cr_return_tax .decimal(7,2), 'cr_return_amt_inc_tax .decimal(7,2), @@ -325,23 +325,23 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'cr_net_loss .decimal(7,2)), Table("inventory", partitionColumns = "inv_date_sk" :: Nil, - 'inv_date_sk .int, - 'inv_item_sk .int, - 'inv_warehouse_sk .int, + 'inv_date_sk .long, + 'inv_item_sk .long, + 'inv_warehouse_sk .long, 'inv_quantity_on_hand .int), Table("store_sales", partitionColumns = "ss_sold_date_sk" :: Nil, - 'ss_sold_date_sk .int, - 'ss_sold_time_sk .int, - 'ss_item_sk .int, - 'ss_customer_sk .int, - 'ss_cdemo_sk .int, - 'ss_hdemo_sk .int, - 'ss_addr_sk .int, - 'ss_store_sk .int, - 'ss_promo_sk .int, - 'ss_ticket_number .int, - 'ss_quantity .int, + 'ss_sold_date_sk .long, + 'ss_sold_time_sk .long, + 'ss_item_sk .long, + 'ss_customer_sk .long, + 'ss_cdemo_sk .long, + 'ss_hdemo_sk .long, + 'ss_addr_sk .long, + 'ss_store_sk .long, + 'ss_promo_sk .long, + 'ss_ticket_number .long, + 'ss_quantity .long, 'ss_wholesale_cost .decimal(7,2), 'ss_list_price .decimal(7,2), 'ss_sales_price .decimal(7,2), @@ -378,25 +378,25 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'sr_net_loss .decimal(7,2)), Table("web_sales", partitionColumns = "ws_sold_date_sk" :: Nil, - 'ws_sold_date_sk .int, - 'ws_sold_time_sk .int, - 'ws_ship_date_sk .int, - 'ws_item_sk .int, - 'ws_bill_customer_sk .int, - 'ws_bill_cdemo_sk .int, - 'ws_bill_hdemo_sk .int, - 'ws_bill_addr_sk .int, - 'ws_ship_customer_sk .int, - 'ws_ship_cdemo_sk .int, - 'ws_ship_hdemo_sk .int, - 'ws_ship_addr_sk .int, - 'ws_web_page_sk .int, - 'ws_web_site_sk .int, - 'ws_ship_mode_sk .int, - 'ws_warehouse_sk .int, - 'ws_promo_sk .int, - 'ws_order_number .int, - 'ws_quantity .int, + 'ws_sold_date_sk .long, + 'ws_sold_time_sk .long, + 'ws_ship_date_sk .long, + 'ws_item_sk .long, + 'ws_bill_customer_sk .long, + 'ws_bill_cdemo_sk .long, + 'ws_bill_hdemo_sk .long, + 'ws_bill_addr_sk .long, + 'ws_ship_customer_sk .long, + 'ws_ship_cdemo_sk .long, + 'ws_ship_hdemo_sk .long, + 'ws_ship_addr_sk .long, + 'ws_web_page_sk .long, + 'ws_web_site_sk .long, + 'ws_ship_mode_sk .long, + 'ws_warehouse_sk .long, + 'ws_promo_sk .long, + 'ws_order_number .long, + 'ws_quantity .long, 'ws_wholesale_cost .decimal(7,2), 'ws_list_price .decimal(7,2), 'ws_sales_price .decimal(7,2), @@ -440,12 +440,12 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'wr_net_loss .decimal(7,2)), Table("call_center", partitionColumns = Nil, - 'cc_call_center_sk .int, + 'cc_call_center_sk .long, 'cc_call_center_id .string, 'cc_rec_start_date .date, 'cc_rec_end_date .date, - 'cc_closed_date_sk .int, - 'cc_open_date_sk .int, + 'cc_closed_date_sk .long, + 'cc_open_date_sk .long, 'cc_name .string, 'cc_class .string, 'cc_employees .int, @@ -473,10 +473,10 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'cc_tax_percentage .decimal(5,2)), Table("catalog_page", partitionColumns = Nil, - 'cp_catalog_page_sk .int, + 'cp_catalog_page_sk .long, 'cp_catalog_page_id .string, - 'cp_start_date_sk .int, - 'cp_end_date_sk .int, + 'cp_start_date_sk .long, + 'cp_end_date_sk .long, 'cp_department .string, 'cp_catalog_number .int, 'cp_catalog_page_number .int, @@ -484,13 +484,13 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'cp_type .string), Table("customer", partitionColumns = Nil, - 'c_customer_sk .int, + 'c_customer_sk .long, 'c_customer_id .string, - 'c_current_cdemo_sk .int, - 'c_current_hdemo_sk .int, - 'c_current_addr_sk .int, - 'c_first_shipto_date_sk .int, - 'c_first_sales_date_sk .int, + 'c_current_cdemo_sk .long, + 'c_current_hdemo_sk .long, + 'c_current_addr_sk .long, + 'c_first_shipto_date_sk .long, + 'c_first_sales_date_sk .long, 'c_salutation .string, 'c_first_name .string, 'c_last_name .string, @@ -504,7 +504,7 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'c_last_review_date .string), Table("customer_address", partitionColumns = Nil, - 'ca_address_sk .int, + 'ca_address_sk .long, 'ca_address_id .string, 'ca_street_number .string, 'ca_street_name .string, @@ -519,7 +519,7 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'ca_location_type .string), Table("customer_demographics", partitionColumns = Nil, - 'cd_demo_sk .int, + 'cd_demo_sk .long, 'cd_gender .string, 'cd_marital_status .string, 'cd_education_status .string, @@ -530,9 +530,9 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'cd_dep_college_count .int), Table("date_dim", partitionColumns = Nil, - 'd_date_sk .int, + 'd_date_sk .long, 'd_date_id .string, - 'd_date .string, + 'd_date .date, 'd_month_seq .int, 'd_week_seq .int, 'd_quarter_seq .int, @@ -560,22 +560,22 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'd_current_year .string), Table("household_demographics", partitionColumns = Nil, - 'hd_demo_sk .int, - 'hd_income_band_sk .int, + 'hd_demo_sk .long, + 'hd_income_band_sk .long, 'hd_buy_potential .string, 'hd_dep_count .int, 'hd_vehicle_count .int), Table("income_band", partitionColumns = Nil, - 'ib_income_band_sk .int, + 'ib_income_band_sk .long, 'ib_lower_bound .int, 'ib_upper_bound .int), Table("item", partitionColumns = Nil, - 'i_item_sk .int, + 'i_item_sk .long, 'i_item_id .string, - 'i_rec_start_date .string, - 'i_rec_end_date .string, + 'i_rec_start_date .date, + 'i_rec_end_date .date, 'i_item_desc .string, 'i_current_price .decimal(7,2), 'i_wholesale_cost .decimal(7,2), @@ -596,11 +596,11 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'i_product_name .string), Table("promotion", partitionColumns = Nil, - 'p_promo_sk .int, + 'p_promo_sk .long, 'p_promo_id .string, - 'p_start_date_sk .int, - 'p_end_date_sk .int, - 'p_item_sk .int, + 'p_start_date_sk .long, + 'p_end_date_sk .long, + 'p_item_sk .long, 'p_cost .decimal(15,2), 'p_response_target .int, 'p_promo_name .string, @@ -617,12 +617,12 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'p_discount_active .string), Table("reason", partitionColumns = Nil, - 'r_reason_sk .int, + 'r_reason_sk .long, 'r_reason_id .string, 'r_reason_desc .string), Table("ship_mode", partitionColumns = Nil, - 'sm_ship_mode_sk .int, + 'sm_ship_mode_sk .long, 'sm_ship_mode_id .string, 'sm_type .string, 'sm_code .string, @@ -630,11 +630,11 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'sm_contract .string), Table("store", partitionColumns = Nil, - 's_store_sk .int, + 's_store_sk .long, 's_store_id .string, - 's_rec_start_date .string, - 's_rec_end_date .string, - 's_closed_date_sk .int, + 's_rec_start_date .date, + 's_rec_end_date .date, + 's_closed_date_sk .long, 's_store_name .string, 's_number_employees .int, 's_floor_space .int, @@ -661,7 +661,7 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 's_tax_precentage .decimal(5,2)), Table("time_dim", partitionColumns = Nil, - 't_time_sk .int, + 't_time_sk .long, 't_time_id .string, 't_time .int, 't_hour .int, @@ -673,7 +673,7 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 't_meal_time .string), Table("warehouse", partitionColumns = Nil, - 'w_warehouse_sk .int, + 'w_warehouse_sk .long, 'w_warehouse_id .string, 'w_warehouse_name .string, 'w_warehouse_sq_ft .int, @@ -689,14 +689,14 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'w_gmt_offset .decimal(5,2)), Table("web_page", partitionColumns = Nil, - 'wp_web_page_sk .int, + 'wp_web_page_sk .long, 'wp_web_page_id .string, 'wp_rec_start_date .date, 'wp_rec_end_date .date, - 'wp_creation_date_sk .int, - 'wp_access_date_sk .int, + 'wp_creation_date_sk .long, + 'wp_access_date_sk .long, 'wp_autogen_flag .string, - 'wp_customer_sk .int, + 'wp_customer_sk .long, 'wp_url .string, 'wp_type .string, 'wp_char_count .int, @@ -705,13 +705,13 @@ class Tables(sqlContext: SQLContext, dsdgenDir: String, scaleFactor: Int) extend 'wp_max_ad_count .int), Table("web_site", partitionColumns = Nil, - 'web_site_sk .int, + 'web_site_sk .long, 'web_site_id .string, 'web_rec_start_date .date, 'web_rec_end_date .date, 'web_name .string, - 'web_open_date_sk .int, - 'web_close_date_sk .int, + 'web_open_date_sk .long, + 'web_close_date_sk .long, 'web_class .string, 'web_manager .string, 'web_mkt_id .int,