Business intelligence made simple
Preserve columns with the same name - closes #43
Andrew Kane45b60dd84736
diff --git a/app/controllers/blazer/dashboards_controller.rb b/app/controllers/blazer/dashboards_controller.rbindex cdc1eb7b..232c9a9d 100644--- a/app/controllers/blazer/dashboards_controller.rb+++ b/app/controllers/blazer/dashboards_controller.rb@@ -37,8 +37,8 @@ module Blazer@data_sources.each do |data_source|query = data_source.smart_variables[var]if query- rows, error, cached_at = data_source.run_statement(query)- ((@smart_vars[var] ||= []).concat(rows.map { |v| v.values.reverse })).uniq!+ columns, rows, error, cached_at = data_source.run_statement(query)+ ((@smart_vars[var] ||= []).concat(rows.map { |v| v.reverse })).uniq!@sql_errors << error if errorendenddiff --git a/app/controllers/blazer/queries_controller.rb b/app/controllers/blazer/queries_controller.rbindex 59ab9d8a..21ed7093 100644--- a/app/controllers/blazer/queries_controller.rb+++ b/app/controllers/blazer/queries_controller.rb@@ -51,8 +51,8 @@ module Blazer@bind_vars.each do |var|query = data_source.smart_variables[var]if query- rows, error, cached_at = data_source.run_statement(query)- @smart_vars[var] = rows.map { |v| v.values.reverse }+ columns, rows, error, cached_at = data_source.run_statement(query)+ @smart_vars[var] = rows.map { |v| v.reverse }@sql_errors << error if errorendend@@ -85,7 +85,7 @@ module Blazeraudit.save!end- @rows, @error, @cached_at = @data_source.run_statement(@statement, user: blazer_user, query: @query, refresh_cache: params[:check])+ @columns, @rows, @error, @cached_at = @data_source.run_statement(@statement, user: blazer_user, query: @query, refresh_cache: params[:check])if @query && @error != Blazer::TIMEOUT_MESSAGE@query.checks.each do |check|@@ -93,11 +93,12 @@ module Blazerendend- @columns = {}+ @first_row = @rows.first || []+ @column_types = []if @rows.any?- @rows.first.each do |key, value|- @columns[key] =- case value+ @columns.each_with_index do |column, i|+ @column_types << (+ case @first_row[i]when Integer"int"when Float@@ -105,20 +106,20 @@ module Blazerelse"string-ins"end+ )endend@filename = @query.name.parameterize if @query-- @min_width_types = (@rows.first || {}).select { |k, v| v.is_a?(Time) || v.is_a?(String) || @data_source.smart_columns[k] }.keys+ @min_width_types = @columns.each_with_index.select { |c, i| @first_row[i].is_a?(Time) || @first_row[i].is_a?(String) || @data_source.smart_columns[c] }@boom = {}- @columns.keys.each do |key|+ @columns.each_with_index do |key, i|query = @data_source.smart_columns[key]if query- values = @rows.map { |r| r[key] }.compact.uniq- rows, error, cached_at = @data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))- @boom[key] = Hash[rows.map(&:values).map { |k, v| [k.to_s, v] }]+ values = @rows.map { |r| r[i] }.compact.uniq+ columns, rows, error, cached_at = @data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))+ @boom[key] = Hash[rows.map { |k, v| [k.to_s, v] }]endend@@ -126,15 +127,17 @@ module Blazer@markers = [][["latitude", "longitude"], ["lat", "lon"]].each do |keys|- if (keys - (@rows.first || {}).keys).empty?+ lat_index = @columns.index(keys.first)+ lon_index = @columns.index(keys.last)+ if lat_index && lon_index@markers =@rows.select do |r|- r[keys.first] && r[keys.last]+ r[lat_index] && r[lon_index]end.map do |r|{- title: r.except(*keys).map{ |k, v| "<strong>#{k}:</strong> #{v}" }.join("<br />").truncate(140),- latitude: r[keys.first],- longitude: r[keys.last]+ title: r.each_with_index.map{ |v, i| i == lat_index || i == lon_index ? nil : "<strong>#{@columns[i]}:</strong> #{v}" }.compact.join("<br />").truncate(140),+ latitude: r[lat_index],+ longitude: r[lon_index]}endend@@ -146,7 +149,7 @@ module Blazerrender layout: falseendformat.csv do- send_data csv_data(@rows), type: "text/csv; charset=utf-8; header=present", disposition: "attachment; filename=\"#{@query.try(:name).try(:parameterize).presence || 'query'}.csv\""+ send_data csv_data(@columns, @rows), type: "text/csv; charset=utf-8; header=present", disposition: "attachment; filename=\"#{@query.try(:name).try(:parameterize).presence || 'query'}.csv\""endendend@@ -181,7 +184,7 @@ module Blazerenddef tables- @tables = Blazer.data_sources[params[:data_source]].tables.keys+ @tables = Blazer.data_sources[params[:data_source]].tablesrender partial: "tables", layout: falseend@@ -224,13 +227,11 @@ module Blazerparams.require(:query).permit(:name, :description, :statement, :data_source)end- def csv_data(rows)+ def csv_data(columns, rows)CSV.generate do |csv|- if rows.any?- csv << rows.first.keys- end+ csv << columnsrows.each do |row|- csv << row.map { |k, v| v.is_a?(Time) ? blazer_time_value(k, v) : v }+ csv << row.each_with_index.map { |v, i| v.is_a?(Time) ? blazer_time_value(columns[i], v) : v }endendenddiff --git a/app/helpers/blazer/base_helper.rb b/app/helpers/blazer/base_helper.rbindex 6fe7e583..7232aa5a 100644--- a/app/helpers/blazer/base_helper.rb+++ b/app/helpers/blazer/base_helper.rb@@ -17,8 +17,8 @@ module Blazerenddef blazer_column_types(columns, rows, boom)- columns.map do |k, _|- v = (rows.find { |r| r[k] } || {})[k]+ columns.each_with_index.map do |k, i|+ v = rows.find { |r| r[i] }[i]if boom[k]"string"elsif v.is_a?(Numeric)diff --git a/app/views/blazer/queries/run.html.erb b/app/views/blazer/queries/run.html.erbindex 1e9cb290..f64f864b 100644--- a/app/views/blazer/queries/run.html.erb+++ b/app/views/blazer/queries/run.html.erb@@ -24,13 +24,13 @@<p class="text-muted"><%= pluralize(@rows.size, "row") %></p><% end %><% if @rows.any? %>- <% values = @rows.first.values %>+ <% values = @rows.first %><% chart_id = SecureRandom.hex %><% column_types = blazer_column_types(@columns, @rows, @boom) %><% chart_options = {id: chart_id, min: nil} %><% series_library = {} %>- <% @columns.keys.select { |k| k.downcase == "target" }.each do |key| %>- <% series_library[key] = {pointStyle: "line", hitRadius: 5, borderColor: "#109618", pointBackgroundColor: "#109618", backgroundColor: "#109618"} %>+ <% @columns.select { |k| k.downcase == "target" }.each_with_index do |key, i| %>+ <% series_library[i] = {pointStyle: "line", hitRadius: 5, borderColor: "#109618", pointBackgroundColor: "#109618", backgroundColor: "#109618"} %><% end %><% if blazer_maps? && @markers.any? %><div id="map" style="height: <%= @only_chart ? 300 : 500 %>px;"></div>@@ -62,32 +62,30 @@map.fitBounds(featureLayer.getBounds());</script><% elsif values.size >= 2 && column_types.compact == ["time"] + (column_types.compact.size - 1).times.map { "numeric" } %>- <% time_k = @columns.keys.first %>- <%= line_chart @columns.keys[1..-1].map{ |k| {name: k, data: @rows.map{|r| [r[time_k], r[k]] }, library: series_library[k]} }, chart_options %>+ <%= line_chart @columns[1..-1].each_with_index.map{ |k, i| {name: k, data: @rows.map{ |r| [r[0], r[i + 1]] }, library: series_library[i]} }, chart_options %><% elsif values.size == 3 && column_types == ["time", "string", "numeric"] %>- <% keys = @columns.keys %>- <%= line_chart @rows.group_by { |r| k = keys[1]; v = r[k]; (@boom[k] || {})[v.to_s] || v }.map { |name, v| {name: name, data: v.map { |v2| [v2[keys[0]], v2[keys[2]]] }, library: series_library[name]} }, chart_options %>+ <%= line_chart @rows.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: name, data: v.map { |v2| [v2[0], v2[2]] }, library: series_library[i]} }, chart_options %><% elsif values.size >= 2 && column_types == ["string"] + (values.size - 1).times.map { "numeric" } %>- <% keys = @columns.keys %>- <%= column_chart (values.size - 1).times.map { |i| name = @columns.keys[i + 1]; {name: name, data: @rows.first(20).map { |r| [(@boom[keys[0]] || {})[r[keys[0]].to_s] || r[keys[0]], r[keys[i + 1]]] } } }, id: chart_id %>+ <%= column_chart (values.size - 1).times.map { |i| name = @columns[i + 1]; {name: name, data: @rows.first(20).map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] } } }, id: chart_id %><% elsif @only_chart %><% if @rows.size == 1 && @rows.first.size == 1 %>- <p style="font-size: 160px;"><%= blazer_format_value(@rows.first.keys.first, @rows.first.values.first) %></p>+ <p style="font-size: 160px;"><%= blazer_format_value(@columns.first, @rows.first.first) %></p><% else %><% @no_chart = true %><% end %><% end %><% unless @only_chart && !@no_chart %>- <% header_width = 100 / @rows.first.keys.size.to_f %>+ <% header_width = 100 / @columns.size.to_f %><div class="results-container">- <% if @columns.keys == ["QUERY PLAN"] %>- <pre><code><%= @rows.map { |r| r["QUERY PLAN"] }.join("\n") %></code></pre>+ <% if @columns == ["QUERY PLAN"] %>+ <pre><code><%= @rows.map { |r| r[0] }.join("\n") %></code></pre><% else %><table class="table results-table" style="margin-bottom: 0;"><thead><tr>- <% @columns.each do |key, type| %>+ <% @columns.each_with_index do |key, i| %>+ <% type = @column_types[i] %><th style="width: <%= header_width %>%;" data-sort="<%= type %>"><div style="min-width: <%= @min_width_types.include?(key) ? 180 : 60 %>px;"><%= key %>@@ -99,7 +97,8 @@<tbody><% @rows.each do |row| %><tr>- <% row.each do |k, v| %>+ <% row.each_with_index do |v, i| %>+ <% k = @columns[i] %><td><% if v.is_a?(Time) %><% v = blazer_time_value(k, v) %>diff --git a/lib/blazer.rb b/lib/blazer.rbindex 8d707902..1bafd2b9 100644--- a/lib/blazer.rb+++ b/lib/blazer.rb@@ -72,7 +72,7 @@ module BlazerActiveSupport::Notifications.instrument("run_check.blazer", check_id: check.id, query_id: check.query.id, state_was: check.state) do |instrument|# try 3 times on timeout errorswhile tries <= 3- rows, error, cached_at = data_sources[check.query.data_source].run_statement(check.query.statement, refresh_cache: true)+ columns, rows, error, cached_at = data_sources[check.query.data_source].run_statement(check.query.statement, refresh_cache: true)if error == Blazer::TIMEOUT_MESSAGERails.logger.info "[blazer timeout] query=#{check.query.name}"tries += 1diff --git a/lib/blazer/data_source.rb b/lib/blazer/data_source.rbindex 9ce2ae9f..c009a3b4 100644--- a/lib/blazer/data_source.rb+++ b/lib/blazer/data_source.rb@@ -58,16 +58,18 @@ module Blazerenddef run_statement(statement, options = {})+ columns = nilrows = nilerror = nilcached_at = nilcache_key = self.cache_key(statement) if cacheif cache && !options[:refresh_cache]value = Blazer.cache.read(cache_key)- rows, cached_at = Marshal.load(value) if value+ columns, rows, cached_at = Marshal.load(value) if valueendunless rows+ columns = []rows = []comment = "blazer"@@ -95,13 +97,10 @@ module Blazerendresult = connection_model.connection.select_all("#{statement} /*#{comment}*/")+ columns = result.columnscast_method = Rails::VERSION::MAJOR < 5 ? :type_cast : :cast_value- result.each do |untyped_row|- row = {}- untyped_row.each do |k, v|- row[k] = result.column_types.empty? ? v : result.column_types[k].send(cast_method, v)- end- rows << row+ result.rows.each do |untyped_row|+ rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| result.column_types[c].send(cast_method, untyped_row[i]) })endrescue ActiveRecord::StatementInvalid => eerror = e.message.sub(/.+ERROR: /, "")@@ -109,10 +108,10 @@ module Blazerendend- Blazer.cache.write(cache_key, Marshal.dump([rows, Time.now]), expires_in: cache.to_f * 60) if !error && cache+ Blazer.cache.write(cache_key, Marshal.dump([columns, rows, Time.now]), expires_in: cache.to_f * 60) if !error && cacheend- [rows, error, cached_at]+ [columns, rows, error, cached_at]enddef clear_cache(statement)@@ -120,7 +119,7 @@ module Blazerenddef cache_key(statement)- ["blazer", "v2", id, Digest::MD5.hexdigest(statement)].join("/")+ ["blazer", "v3", id, Digest::MD5.hexdigest(statement)].join("/")enddef schemas@@ -129,8 +128,8 @@ module Blazerenddef tables- rows, error, cached_at = run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name, column_name, ordinal_position, data_type FROM information_schema.columns WHERE table_schema IN (?)", schemas]))- Hash[rows.group_by { |r| r["table_name"] }.map { |t, f| [t, f.sort_by { |f| f["ordinal_position"] }.map { |f| f.slice("column_name", "data_type") }] }.sort_by { |t, _f| t }]+ columns, rows, error, cached_at = run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name, column_name, ordinal_position, data_type FROM information_schema.columns WHERE table_schema IN (?)", schemas]))+ rows.map(&:first)enddef postgresql?