Business intelligence made simple
Added support for Apache Ignite
Andrew Kane8a63f0dd190b
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 210e0a87..ce1c827e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,7 @@+## 2.4.2 (unreleased)++- Added support for Apache Ignite+## 2.4.1 (2021-01-25)- Added cohorts for MySQLdiff --git a/README.md b/README.mdindex 785f306f..8e548a1e 100644--- a/README.md+++ b/README.md@@ -567,6 +567,7 @@ data_sources:- [Amazon Redshift](#amazon-redshift)- [Apache Drill](#apache-drill)- [Apache Hive](#apache-hive)+- [Apache Ignite](#apache-ignite) [master]- [Apache Spark](#apache-spark)- [Cassandra](#cassandra)- [Druid](#druid)@@ -642,6 +643,19 @@ data_sources:Use a read-only user. Requires [HiveServer2](https://cwiki.apache.org/confluence/display/Hive/Setting+Up+HiveServer2).+### Apache Ignite++Set:++```yml+data_sources:+ my_source:+ adapter: ignite+ url: http://localhost:8080+```++Use a read-only user. Requires the [REST API](https://ignite.apache.org/docs/latest/restapi) to be enabled.+### Apache SparkAdd [hexspace](https://github.com/ankane/hexspace) to your Gemfile and set:diff --git a/lib/blazer.rb b/lib/blazer.rbindex d1f72bf7..9928c9a1 100644--- a/lib/blazer.rb+++ b/lib/blazer.rb@@ -19,6 +19,7 @@ require "blazer/adapters/drill_adapter"require "blazer/adapters/druid_adapter"require "blazer/adapters/elasticsearch_adapter"require "blazer/adapters/hive_adapter"+require "blazer/adapters/ignite_adapter"require "blazer/adapters/influxdb_adapter"require "blazer/adapters/mongodb_adapter"require "blazer/adapters/neo4j_adapter"@@ -242,6 +243,7 @@ Blazer.register_adapter "drill", Blazer::Adapters::DrillAdapterBlazer.register_adapter "druid", Blazer::Adapters::DruidAdapterBlazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapterBlazer.register_adapter "hive", Blazer::Adapters::HiveAdapter+Blazer.register_adapter "ignite", Blazer::Adapters::IgniteAdapterBlazer.register_adapter "influxdb", Blazer::Adapters::InfluxdbAdapterBlazer.register_adapter "neo4j", Blazer::Adapters::Neo4jAdapterBlazer.register_adapter "presto", Blazer::Adapters::PrestoAdapterdiff --git a/lib/blazer/adapters/ignite_adapter.rb b/lib/blazer/adapters/ignite_adapter.rbnew file mode 100644index 00000000..70837bbc--- /dev/null+++ b/lib/blazer/adapters/ignite_adapter.rb@@ -0,0 +1,76 @@+module Blazer+ module Adapters+ class IgniteAdapter < BaseAdapter+ def run_statement(statement, comment)+ columns = []+ rows = []+ error = nil++ uri = URI("#{settings["url"].chomp("/")}/ignite")+ uri.query = URI.encode_www_form(+ "cmd" => "qryfldexe",+ "pageSize" => 1_000_000_000, # TODO paginate+ "cacheName" => "PUBLIC",+ "qry" => "#{statement} /*#{comment}*/"+ )++ req = Net::HTTP::Get.new(uri)++ options = {+ use_ssl: uri.scheme == "https",+ open_timeout: 3,+ read_timeout: 30+ }++ begin+ # use Net::HTTP instead of soda-ruby for types and better error messages+ res = Net::HTTP.start(uri.hostname, uri.port, options) do |http|+ http.request(req)+ end++ if res.is_a?(Net::HTTPSuccess)+ body = JSON.parse(res.body)++ if body["successStatus"] == 0+ # TODO typecast+ columns = body["response"]["fieldsMetadata"].map { |v| v["fieldName"] } if columns.empty?+ rows = body["response"]["items"]+ else+ error = body["error"]+ end+ else+ error = JSON.parse(res.body)["message"] rescue "Bad response: #{res.code}"+ end+ rescue => e+ error = e.message+ end++ [columns, rows, error]+ end++ def preview_statement+ "SELECT * FROM {table} LIMIT 10"+ end++ def tables+ sql = "SELECT table_schema, table_name FROM information_schema.tables WHERE table_schema NOT IN ('INFORMATION_SCHEMA', 'SYS')"+ result = data_source.run_statement(sql)+ result.rows.reject { |row| row[1].start_with?("__") }.map do |row|+ (row[0] == default_schema ? row[1] : "#{row[0]}.#{row[1]}").downcase+ end+ end++ # TODO figure out error+ # Table `__T0` can be accessed only within Ignite query context.+ # def schema+ # sql = "SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM information_schema.columns WHERE table_schema NOT IN ('INFORMATION_SCHEMA', 'SYS')"+ # result = data_source.run_statement(sql)+ # result.rows.group_by { |r| [r[0], r[1]] }.map { |k, vs| {schema: k[0], table: k[1], columns: vs.sort_by { |v| v[2] }.map { |v| {name: v[2], data_type: v[3]} }} }.sort_by { |t| [t[:schema] == default_schema ? "" : t[:schema], t[:table]] }+ # end++ def default_schema+ "PUBLIC"+ end+ end+ end+end