The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve dataset creation for Bigquery (#76031)
technomancy28b1a1507e76
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.cljindex 59e9de527cd..26cd73deee3 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj@@ -339,20 +339,20 @@(recur (dec num-retries))(throw e))))))))-(defn delete-old-datasets!- []+(defn delete-old-datasets! [](let [all-outdated (execute!- (str "(SELECT `name` FROM `%s.metabase_test_tracking.datasets` WHERE `accessed_at` < TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -2 hour))"+ (str "(SELECT `name` FROM `%s.metabase_test_tracking.datasets`"+ " WHERE `accessed_at` < TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -14 day))"" UNION ALL ""(select schema_name from `%s`.INFORMATION_SCHEMA.SCHEMATA dwhere d.schema_name not in (select name from `%s.metabase_test_tracking.datasets`)and d.schema_name like 'sha_%%'- and creation_time < TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -2 hour))")+ and creation_time < TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -14 day))")(project-id)(project-id)(project-id))](doseq [outdated (map first all-outdated)]- (log/info (u/format-color 'blue "Deleting temporary dataset more than two days old: %s`." outdated))+ (log/info (u/format-color 'blue "Deleting temporary dataset: %s`." outdated))(destroy-dataset! outdated))));;; ---------------------- Workspace isolation orphan cleanup ----------------------@@ -526,26 +526,28 @@(test-dataset-id db-def)(tx/tracking-access-note)])))+(defn- get-existing-tables [dataset-id]+ (let [sql (format "SELECT table_name FROM %s.%s.INFORMATION_SCHEMA.TABLES"+ (project-id) dataset-id)]+ (set (map :table_name (execute! sql)))))+(defmethod tx/create-db! :bigquery-cloud-sdk[driver {:keys [database-name table-definitions options] :as db-def} & _]{:pre [(seq database-name) (sequential? table-definitions)]}- (delete-old-datasets-if-needed!)+ ;; re-enable this again once things are stable; for now let's just completely+ ;; excise this potential source of unreliability+ (comment (delete-old-datasets-if-needed!))(let [dataset-id (test-dataset-id db-def)]- (if (database-exists?! db-def)- (log/info (u/format-color 'blue "Dataset already exists %s, not loading db" (pr-str dataset-id)))- (try- (log/infof "Creating dataset %s..." (pr-str dataset-id))- (create-dataset! dataset-id)- ;; now create tables and load data.- (doseq [tabledef table-definitions]- (load-tabledef! dataset-id tabledef))- (doseq [native-ddl (:native-ddl options)]- (apply execute! (sql.tx/compile-native-ddl driver native-ddl)))- (log/info (u/format-color 'green "Successfully created %s." (pr-str dataset-id)))- (catch Throwable e- (log/error (u/format-color 'red "Failed to load BigQuery dataset %s." (pr-str dataset-id)))- (log/error (u/pprint-to-str 'red (Throwable->map e)))- (throw e))))))+ (when-not (database-exists?! db-def)+ (create-dataset! dataset-id))+ ;; now create tables and load data.+ (let [existing-tables (get-existing-tables dataset-id)]+ (doseq [tabledef table-definitions+ :when (not (existing-tables (:table-name tabledef)))]+ (load-tabledef! dataset-id tabledef)))+ (doseq [native-ddl (:native-ddl options)]+ (apply execute! (sql.tx/compile-native-ddl driver native-ddl)))+ (log/info (u/format-color 'green "Successfully created %s." (pr-str dataset-id)))))(defmethod tx/destroy-db! :bigquery-cloud-sdk[_ db-def]