Skip to content

Business intelligence made simple

BigQuery cleanup

Andrew Kane325207c9439c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 29344a41..7edc6e40 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 1.7.10 [unreleased]
+
+- Added support for Google BigQuery
+
## 1.7.9
- Added beta support for Apache Drill
diff --git a/README.md b/README.md
index b626e8ac..efce6aa3 100644
--- a/README.md
+++ b/README.md
@@ -392,7 +392,7 @@ data_sources:
- [Redshift](#redshift)
- [Presto](#presto)
- [Apache Drill](#apache-drill)
-- [Google BigQuery](#google-bigquery)
+- [Google BigQuery](#google-bigquery-master)
- [MongoDB](#mongodb-1)
- [Elasticsearch](#elasticsearch) [beta]
@@ -485,7 +485,7 @@ data_sources:
url: http://hostname:8047
```
-### Google BigQuery
+### Google BigQuery [master]
Add [google-cloud-bigquery](https://github.com/GoogleCloudPlatform/google-cloud-ruby/tree/master/google-cloud-bigquery) to your Gemfile and set:
@@ -493,9 +493,8 @@ Add [google-cloud-bigquery](https://github.com/GoogleCloudPlatform/google-cloud-
data_sources:
my_source:
adapter: bigquery
- url: "" # required for blazer
- project: "your_project"
- keyfile: "json credentials file"
+ project: your-project
+ keyfile: path/to/keyfile.json
```
### MongoDB
diff --git a/app/views/blazer/queries/show.html.erb b/app/views/blazer/queries/show.html.erb
index 1467a1e1..b4e5ebe0 100644
--- a/app/views/blazer/queries/show.html.erb
+++ b/app/views/blazer/queries/show.html.erb
@@ -62,7 +62,7 @@
</script>
<% end %>
-<% if %w[sql presto drill].include?(Blazer.data_sources[@query.data_source].adapter) %>
+<% if %w[sql presto drill bigquery].include?(Blazer.data_sources[@query.data_source].adapter) %>
<script>
// do not highlight really long queries
// this can lead to performance issues
diff --git a/lib/blazer/adapters/bigquery_adapter.rb b/lib/blazer/adapters/bigquery_adapter.rb
index f6564656..0fc0ff9d 100644
--- a/lib/blazer/adapters/bigquery_adapter.rb
+++ b/lib/blazer/adapters/bigquery_adapter.rb
@@ -5,28 +5,30 @@ module Blazer
columns = []
rows = []
error = nil
+
begin
- results = bigquery.query(statement, timeout: timeout_ms)
+ results = bigquery.query(statement, timeout: 30000) # ms
columns = results.first.keys.map(&:to_s) if results.size > 0
rows = results.map(&:values)
- rescue StandardError => e
+ rescue => e
error = e.message
end
+
[columns, rows, error]
end
def tables
- table_refs.map{|t| "#{t.project_id}.#{t.dataset_id}.#{t.table_id}" }
+ table_refs.map { |t| "#{t.project_id}.#{t.dataset_id}.#{t.table_id}" }
end
def schema
- table_refs.map{|table_ref|
+ table_refs.map do |table_ref|
{
schema: table_ref.dataset_id,
table: table_ref.table_id,
columns: table_columns(table_ref)
}
- }
+ end
end
def preview_statement
@@ -36,38 +38,23 @@ module Blazer
private
def bigquery
- @bigquery ||= connect!
- end
-
- def connect!
- require "google/cloud/bigquery"
- params = { project: settings["project"], keyfile: settings["keyfile"] }
- @bigquery = Google::Cloud::Bigquery.new(params)
- ::Google::Apis.logger.level = Logger::INFO
- @bigquery
+ @bigquery ||= begin
+ require "google/cloud/bigquery"
+ Google::Cloud::Bigquery.new(
+ project: settings["project"],
+ keyfile: settings["keyfile"]
+ )
+ end
end
def table_refs
- bigquery
- .datasets
- .map(&:tables)
- .flat_map { |table_list| table_list.map(&:table_ref) }
+ bigquery.datasets.map(&:tables).flat_map { |table_list| table_list.map(&:table_ref) }
end
def table_columns(table_ref)
- schema =
- bigquery
- .service
- .get_table(table_ref.dataset_id, table_ref.table_id)
- .schema
+ schema = bigquery.service.get_table(table_ref.dataset_id, table_ref.table_id).schema
return [] if schema.nil?
- schema
- .fields
- .map { |field| { name: field.name, data_type: field.type} }
- end
-
- def timeout_ms
- 30 * 1000 # 30 seconds
+ schema.fields.map { |field| {name: field.name, data_type: field.type} }
end
end
end
diff --git a/lib/blazer/data_source.rb b/lib/blazer/data_source.rb
index 44089619..fc3ebdff 100644
--- a/lib/blazer/data_source.rb
+++ b/lib/blazer/data_source.rb
@@ -12,7 +12,7 @@ module Blazer
@id = id
@settings = settings
- unless settings["url"] || Rails.env.development?
+ unless settings["url"] || Rails.env.development? || settings["adapter"] == "bigquery"
raise Blazer::Error, "Empty url for data source: #{id}"
end