Business intelligence made simple
BigQuery cleanup
Andrew Kane325207c9439c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 29344a41..7edc6e40 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,7 @@+## 1.7.10 [unreleased]++- Added support for Google BigQuery+## 1.7.9- Added beta support for Apache Drilldiff --git a/README.md b/README.mdindex b626e8ac..efce6aa3 100644--- a/README.md+++ b/README.md@@ -392,7 +392,7 @@ data_sources:- [Redshift](#redshift)- [Presto](#presto)- [Apache Drill](#apache-drill)-- [Google BigQuery](#google-bigquery)+- [Google BigQuery](#google-bigquery-master)- [MongoDB](#mongodb-1)- [Elasticsearch](#elasticsearch) [beta]@@ -485,7 +485,7 @@ data_sources:url: http://hostname:8047```-### Google BigQuery+### Google BigQuery [master]Add [google-cloud-bigquery](https://github.com/GoogleCloudPlatform/google-cloud-ruby/tree/master/google-cloud-bigquery) to your Gemfile and set:@@ -493,9 +493,8 @@ Add [google-cloud-bigquery](https://github.com/GoogleCloudPlatform/google-cloud-data_sources:my_source:adapter: bigquery- url: "" # required for blazer- project: "your_project"- keyfile: "json credentials file"+ project: your-project+ keyfile: path/to/keyfile.json```### MongoDBdiff --git a/app/views/blazer/queries/show.html.erb b/app/views/blazer/queries/show.html.erbindex 1467a1e1..b4e5ebe0 100644--- a/app/views/blazer/queries/show.html.erb+++ b/app/views/blazer/queries/show.html.erb@@ -62,7 +62,7 @@</script><% end %>-<% if %w[sql presto drill].include?(Blazer.data_sources[@query.data_source].adapter) %>+<% if %w[sql presto drill bigquery].include?(Blazer.data_sources[@query.data_source].adapter) %><script>// do not highlight really long queries// this can lead to performance issuesdiff --git a/lib/blazer/adapters/bigquery_adapter.rb b/lib/blazer/adapters/bigquery_adapter.rbindex f6564656..0fc0ff9d 100644--- a/lib/blazer/adapters/bigquery_adapter.rb+++ b/lib/blazer/adapters/bigquery_adapter.rb@@ -5,28 +5,30 @@ module Blazercolumns = []rows = []error = nil+begin- results = bigquery.query(statement, timeout: timeout_ms)+ results = bigquery.query(statement, timeout: 30000) # mscolumns = results.first.keys.map(&:to_s) if results.size > 0rows = results.map(&:values)- rescue StandardError => e+ rescue => eerror = e.messageend+[columns, rows, error]enddef tables- table_refs.map{|t| "#{t.project_id}.#{t.dataset_id}.#{t.table_id}" }+ table_refs.map { |t| "#{t.project_id}.#{t.dataset_id}.#{t.table_id}" }enddef schema- table_refs.map{|table_ref|+ table_refs.map do |table_ref|{schema: table_ref.dataset_id,table: table_ref.table_id,columns: table_columns(table_ref)}- }+ endenddef preview_statement@@ -36,38 +38,23 @@ module Blazerprivatedef bigquery- @bigquery ||= connect!- end-- def connect!- require "google/cloud/bigquery"- params = { project: settings["project"], keyfile: settings["keyfile"] }- @bigquery = Google::Cloud::Bigquery.new(params)- ::Google::Apis.logger.level = Logger::INFO- @bigquery+ @bigquery ||= begin+ require "google/cloud/bigquery"+ Google::Cloud::Bigquery.new(+ project: settings["project"],+ keyfile: settings["keyfile"]+ )+ endenddef table_refs- bigquery- .datasets- .map(&:tables)- .flat_map { |table_list| table_list.map(&:table_ref) }+ bigquery.datasets.map(&:tables).flat_map { |table_list| table_list.map(&:table_ref) }enddef table_columns(table_ref)- schema =- bigquery- .service- .get_table(table_ref.dataset_id, table_ref.table_id)- .schema+ schema = bigquery.service.get_table(table_ref.dataset_id, table_ref.table_id).schemareturn [] if schema.nil?- schema- .fields- .map { |field| { name: field.name, data_type: field.type} }- end-- def timeout_ms- 30 * 1000 # 30 seconds+ schema.fields.map { |field| {name: field.name, data_type: field.type} }endendenddiff --git a/lib/blazer/data_source.rb b/lib/blazer/data_source.rbindex 44089619..fc3ebdff 100644--- a/lib/blazer/data_source.rb+++ b/lib/blazer/data_source.rb@@ -12,7 +12,7 @@ module Blazer@id = id@settings = settings- unless settings["url"] || Rails.env.development?+ unless settings["url"] || Rails.env.development? || settings["adapter"] == "bigquery"raise Blazer::Error, "Empty url for data source: #{id}"end