Business intelligence made simple
add bigquery adapter (#138)
Pedro Carmonab72b74b3036e
diff --git a/README.md b/README.mdindex 0dc10ee7..b626e8ac 100644--- a/README.md+++ b/README.md@@ -392,6 +392,7 @@ data_sources:- [Redshift](#redshift)- [Presto](#presto)- [Apache Drill](#apache-drill)+- [Google BigQuery](#google-bigquery)- [MongoDB](#mongodb-1)- [Elasticsearch](#elasticsearch) [beta]@@ -484,6 +485,19 @@ data_sources:url: http://hostname:8047```+### Google BigQuery++Add [google-cloud-bigquery](https://github.com/GoogleCloudPlatform/google-cloud-ruby/tree/master/google-cloud-bigquery) to your Gemfile and set:++```yml+data_sources:+ my_source:+ adapter: bigquery+ url: "" # required for blazer+ project: "your_project"+ keyfile: "json credentials file"+```+### MongoDBAdd [mongo](https://github.com/mongodb/mongo-ruby-driver) to your Gemfile and set:diff --git a/lib/blazer.rb b/lib/blazer.rbindex 13aa400e..cd08afbb 100644--- a/lib/blazer.rb+++ b/lib/blazer.rb@@ -7,6 +7,7 @@ require "blazer/data_source"require "blazer/result"require "blazer/run_statement"require "blazer/adapters/base_adapter"+require "blazer/adapters/bigquery_adapter"require "blazer/adapters/drill_adapter"require "blazer/adapters/elasticsearch_adapter"require "blazer/adapters/mongodb_adapter"@@ -160,6 +161,7 @@ module BlazerendBlazer.register_adapter "drill", Blazer::Adapters::DrillAdapter+Blazer.register_adapter "bigquery", Blazer::Adapters::BigQueryAdapterBlazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapterBlazer.register_adapter "mongodb", Blazer::Adapters::MongodbAdapterBlazer.register_adapter "presto", Blazer::Adapters::PrestoAdapterdiff --git a/lib/blazer/adapters/bigquery_adapter.rb b/lib/blazer/adapters/bigquery_adapter.rbnew file mode 100644index 00000000..f6564656--- /dev/null+++ b/lib/blazer/adapters/bigquery_adapter.rb@@ -0,0 +1,74 @@+module Blazer+ module Adapters+ class BigQueryAdapter < BaseAdapter+ def run_statement(statement, comment)+ columns = []+ rows = []+ error = nil+ begin+ results = bigquery.query(statement, timeout: timeout_ms)+ columns = results.first.keys.map(&:to_s) if results.size > 0+ rows = results.map(&:values)+ rescue StandardError => e+ error = e.message+ end+ [columns, rows, error]+ end++ def tables+ table_refs.map{|t| "#{t.project_id}.#{t.dataset_id}.#{t.table_id}" }+ end++ def schema+ table_refs.map{|table_ref|+ {+ schema: table_ref.dataset_id,+ table: table_ref.table_id,+ columns: table_columns(table_ref)+ }+ }+ end++ def preview_statement+ "SELECT * FROM `{table}` LIMIT 10"+ end++ private++ def bigquery+ @bigquery ||= connect!+ end++ def connect!+ require "google/cloud/bigquery"+ params = { project: settings["project"], keyfile: settings["keyfile"] }+ @bigquery = Google::Cloud::Bigquery.new(params)+ ::Google::Apis.logger.level = Logger::INFO+ @bigquery+ end++ def table_refs+ bigquery+ .datasets+ .map(&:tables)+ .flat_map { |table_list| table_list.map(&:table_ref) }+ end++ def table_columns(table_ref)+ schema =+ bigquery+ .service+ .get_table(table_ref.dataset_id, table_ref.table_id)+ .schema+ return [] if schema.nil?+ schema+ .fields+ .map { |field| { name: field.name, data_type: field.type} }+ end++ def timeout_ms+ 30 * 1000 # 30 seconds+ end+ end+ end+end