Business intelligence made simple
Added experimental support for the Socrata Open Data API (SODA)
Andrew Kane37864dbb028e
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a3eece91..8fa30d5a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,7 @@+## 2.2.2 (unreleased)++- Added experimental support for the Socrata Open Data API (SODA)+## 2.2.1 (2019-10-08)- Added support for Sprockets 4diff --git a/README.md b/README.mdindex 3b786602..3ceef979 100644--- a/README.md+++ b/README.md@@ -480,6 +480,7 @@ data_sources:- [PostgreSQL](#postgresql-1)- [Presto](#presto)- [Salesforce](#salesforce-experimental)+- [Socrata Open Data API (SODA)](#socrata-open-data-api-soda-experimental-master) [experimental, master]- [Snowflake](#snowflake)- [SQLite](#sqlite)- [SQL Server](#sql-server)@@ -653,6 +654,18 @@ SALESFORCE_API_VERSION="41.0"Supports [SOQL](https://developer.salesforce.com/docs/atlas.en-us.soql_sosl.meta/soql_sosl/sforce_api_calls_soql.htm)+### Socrata Open Data API (SODA) [experimental, master]++Set:++```yml+data_sources:+ my_source:+ adapter: soda+ url: https://soda.demo.socrata.com/resource/4tka-6guv.json+ app_token: ...+```+### SnowflakeFirst, install ODBC. For Homebrew, use:diff --git a/app/helpers/blazer/base_helper.rb b/app/helpers/blazer/base_helper.rbindex 0adce900..9ca9ea5a 100644--- a/app/helpers/blazer/base_helper.rb+++ b/app/helpers/blazer/base_helper.rb@@ -14,7 +14,7 @@ module Blazerdef blazer_format_value(key, value)if value.is_a?(Numeric) && !key.to_s.end_with?("id") && !key.to_s.start_with?("id")number_with_delimiter(value)- elsif value =~ BLAZER_URL_REGEX+ elsif value.is_a?(String) && value =~ BLAZER_URL_REGEX# see if image or linkif Blazer.images && (key.include?("image") || BLAZER_IMAGE_EXT.include?(value.split(".").last.split("?").first.try(:downcase)))link_to value, target: "_blank" dodiff --git a/lib/blazer.rb b/lib/blazer.rbindex 8d6eb001..c906e0f4 100644--- a/lib/blazer.rb+++ b/lib/blazer.rb@@ -22,6 +22,7 @@ require "blazer/adapters/mongodb_adapter"require "blazer/adapters/neo4j_adapter"require "blazer/adapters/presto_adapter"require "blazer/adapters/salesforce_adapter"+require "blazer/adapters/soda_adapter"require "blazer/adapters/sql_adapter"require "blazer/adapters/snowflake_adapter"@@ -223,5 +224,6 @@ Blazer.register_adapter "neo4j", Blazer::Adapters::Neo4jAdapterBlazer.register_adapter "presto", Blazer::Adapters::PrestoAdapterBlazer.register_adapter "mongodb", Blazer::Adapters::MongodbAdapterBlazer.register_adapter "salesforce", Blazer::Adapters::SalesforceAdapter+Blazer.register_adapter "soda", Blazer::Adapters::SodaAdapterBlazer.register_adapter "sql", Blazer::Adapters::SqlAdapterBlazer.register_adapter "snowflake", Blazer::Adapters::SnowflakeAdapterdiff --git a/lib/blazer/adapters/soda_adapter.rb b/lib/blazer/adapters/soda_adapter.rbnew file mode 100644index 00000000..65ff226e--- /dev/null+++ b/lib/blazer/adapters/soda_adapter.rb@@ -0,0 +1,76 @@+module Blazer+ module Adapters+ class SodaAdapter < BaseAdapter+ def run_statement(statement, comment)+ columns = []+ rows = []+ error = nil++ # remove comments manually+ statement = statement.gsub(/--.+/, "")+ # only supports single line /* */ comments+ # regex not perfect, but should be good enough+ statement = statement.gsub(/\/\*.+\*\//, "")++ # remove trailing semicolon+ statement = statement.sub(/;\s*\z/, "")++ # remove whitespace+ statement = statement.squish++ uri = URI(settings["url"])+ uri.query = URI.encode_www_form("$query" => statement)++ req = Net::HTTP::Get.new(uri)+ req["X-App-Token"] = settings["app_token"] if settings["app_token"]++ options = {+ use_ssl: uri.scheme == "https",+ open_timeout: 3,+ read_timeout: 30+ }++ begin+ # use Net::HTTP instead of soda-ruby for types and better error messages+ res = Net::HTTP.start(uri.hostname, uri.port, options) do |http|+ http.request(req)+ end++ if res.is_a?(Net::HTTPSuccess)+ body = JSON.parse(res.body)++ columns = JSON.parse(res["x-soda2-fields"])+ column_types = columns.zip(JSON.parse(res["x-soda2-types"])).to_h++ columns.reject! { |f| f.start_with?(":@") }+ # rows can be missing some keys in JSON, so need to map by column+ rows = body.map { |r| columns.map { |c| r[c] } }++ columns.each_with_index do |column, i|+ if column_types[column] == "number"+ rows.each do |row|+ v = row[i].to_f+ row[i] = v == v.to_i ? v.to_i : v+ end+ end+ end+ else+ error = JSON.parse(res.body)["message"] rescue "Bad response: #{res.code}"+ end+ rescue => e+ error = e.message+ end++ [columns, rows, error]+ end++ def preview_statement+ "SELECT * LIMIT 10"+ end++ def tables+ ["all"]+ end+ end+ end+end