Skip to content

Business intelligence made simple

Added support for Apache Ignite

Andrew Kane8a63f0dd190b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 210e0a87..ce1c827e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 2.4.2 (unreleased)
+
+- Added support for Apache Ignite
+
## 2.4.1 (2021-01-25)
- Added cohorts for MySQL
diff --git a/README.md b/README.md
index 785f306f..8e548a1e 100644
--- a/README.md
+++ b/README.md
@@ -567,6 +567,7 @@ data_sources:
- [Amazon Redshift](#amazon-redshift)
- [Apache Drill](#apache-drill)
- [Apache Hive](#apache-hive)
+- [Apache Ignite](#apache-ignite) [master]
- [Apache Spark](#apache-spark)
- [Cassandra](#cassandra)
- [Druid](#druid)
@@ -642,6 +643,19 @@ data_sources:
Use a read-only user. Requires [HiveServer2](https://cwiki.apache.org/confluence/display/Hive/Setting+Up+HiveServer2).
+### Apache Ignite
+
+Set:
+
+```yml
+data_sources:
+ my_source:
+ adapter: ignite
+ url: http://localhost:8080
+```
+
+Use a read-only user. Requires the [REST API](https://ignite.apache.org/docs/latest/restapi) to be enabled.
+
### Apache Spark
Add [hexspace](https://github.com/ankane/hexspace) to your Gemfile and set:
diff --git a/lib/blazer.rb b/lib/blazer.rb
index d1f72bf7..9928c9a1 100644
--- a/lib/blazer.rb
+++ b/lib/blazer.rb
@@ -19,6 +19,7 @@ require "blazer/adapters/drill_adapter"
require "blazer/adapters/druid_adapter"
require "blazer/adapters/elasticsearch_adapter"
require "blazer/adapters/hive_adapter"
+require "blazer/adapters/ignite_adapter"
require "blazer/adapters/influxdb_adapter"
require "blazer/adapters/mongodb_adapter"
require "blazer/adapters/neo4j_adapter"
@@ -242,6 +243,7 @@ Blazer.register_adapter "drill", Blazer::Adapters::DrillAdapter
Blazer.register_adapter "druid", Blazer::Adapters::DruidAdapter
Blazer.register_adapter "elasticsearch", Blazer::Adapters::ElasticsearchAdapter
Blazer.register_adapter "hive", Blazer::Adapters::HiveAdapter
+Blazer.register_adapter "ignite", Blazer::Adapters::IgniteAdapter
Blazer.register_adapter "influxdb", Blazer::Adapters::InfluxdbAdapter
Blazer.register_adapter "neo4j", Blazer::Adapters::Neo4jAdapter
Blazer.register_adapter "presto", Blazer::Adapters::PrestoAdapter
diff --git a/lib/blazer/adapters/ignite_adapter.rb b/lib/blazer/adapters/ignite_adapter.rb
new file mode 100644
index 00000000..70837bbc
--- /dev/null
+++ b/lib/blazer/adapters/ignite_adapter.rb
@@ -0,0 +1,76 @@
+module Blazer
+ module Adapters
+ class IgniteAdapter < BaseAdapter
+ def run_statement(statement, comment)
+ columns = []
+ rows = []
+ error = nil
+
+ uri = URI("#{settings["url"].chomp("/")}/ignite")
+ uri.query = URI.encode_www_form(
+ "cmd" => "qryfldexe",
+ "pageSize" => 1_000_000_000, # TODO paginate
+ "cacheName" => "PUBLIC",
+ "qry" => "#{statement} /*#{comment}*/"
+ )
+
+ req = Net::HTTP::Get.new(uri)
+
+ options = {
+ use_ssl: uri.scheme == "https",
+ open_timeout: 3,
+ read_timeout: 30
+ }
+
+ begin
+ # use Net::HTTP instead of soda-ruby for types and better error messages
+ res = Net::HTTP.start(uri.hostname, uri.port, options) do |http|
+ http.request(req)
+ end
+
+ if res.is_a?(Net::HTTPSuccess)
+ body = JSON.parse(res.body)
+
+ if body["successStatus"] == 0
+ # TODO typecast
+ columns = body["response"]["fieldsMetadata"].map { |v| v["fieldName"] } if columns.empty?
+ rows = body["response"]["items"]
+ else
+ error = body["error"]
+ end
+ else
+ error = JSON.parse(res.body)["message"] rescue "Bad response: #{res.code}"
+ end
+ rescue => e
+ error = e.message
+ end
+
+ [columns, rows, error]
+ end
+
+ def preview_statement
+ "SELECT * FROM {table} LIMIT 10"
+ end
+
+ def tables
+ sql = "SELECT table_schema, table_name FROM information_schema.tables WHERE table_schema NOT IN ('INFORMATION_SCHEMA', 'SYS')"
+ result = data_source.run_statement(sql)
+ result.rows.reject { |row| row[1].start_with?("__") }.map do |row|
+ (row[0] == default_schema ? row[1] : "#{row[0]}.#{row[1]}").downcase
+ end
+ end
+
+ # TODO figure out error
+ # Table `__T0` can be accessed only within Ignite query context.
+ # def schema
+ # sql = "SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM information_schema.columns WHERE table_schema NOT IN ('INFORMATION_SCHEMA', 'SYS')"
+ # result = data_source.run_statement(sql)
+ # result.rows.group_by { |r| [r[0], r[1]] }.map { |k, vs| {schema: k[0], table: k[1], columns: vs.sort_by { |v| v[2] }.map { |v| {name: v[2], data_type: v[3]} }} }.sort_by { |t| [t[:schema] == default_schema ? "" : t[:schema], t[:table]] }
+ # end
+
+ def default_schema
+ "PUBLIC"
+ end
+ end
+ end
+end