Skip to content

Business intelligence made simple

Preserve columns with the same name - closes #43

Andrew Kane45b60dd84736
diff --git a/app/controllers/blazer/dashboards_controller.rb b/app/controllers/blazer/dashboards_controller.rb
index cdc1eb7b..232c9a9d 100644
--- a/app/controllers/blazer/dashboards_controller.rb
+++ b/app/controllers/blazer/dashboards_controller.rb
@@ -37,8 +37,8 @@ module Blazer
@data_sources.each do |data_source|
query = data_source.smart_variables[var]
if query
- rows, error, cached_at = data_source.run_statement(query)
- ((@smart_vars[var] ||= []).concat(rows.map { |v| v.values.reverse })).uniq!
+ columns, rows, error, cached_at = data_source.run_statement(query)
+ ((@smart_vars[var] ||= []).concat(rows.map { |v| v.reverse })).uniq!
@sql_errors << error if error
end
end
diff --git a/app/controllers/blazer/queries_controller.rb b/app/controllers/blazer/queries_controller.rb
index 59ab9d8a..21ed7093 100644
--- a/app/controllers/blazer/queries_controller.rb
+++ b/app/controllers/blazer/queries_controller.rb
@@ -51,8 +51,8 @@ module Blazer
@bind_vars.each do |var|
query = data_source.smart_variables[var]
if query
- rows, error, cached_at = data_source.run_statement(query)
- @smart_vars[var] = rows.map { |v| v.values.reverse }
+ columns, rows, error, cached_at = data_source.run_statement(query)
+ @smart_vars[var] = rows.map { |v| v.reverse }
@sql_errors << error if error
end
end
@@ -85,7 +85,7 @@ module Blazer
audit.save!
end
- @rows, @error, @cached_at = @data_source.run_statement(@statement, user: blazer_user, query: @query, refresh_cache: params[:check])
+ @columns, @rows, @error, @cached_at = @data_source.run_statement(@statement, user: blazer_user, query: @query, refresh_cache: params[:check])
if @query && @error != Blazer::TIMEOUT_MESSAGE
@query.checks.each do |check|
@@ -93,11 +93,12 @@ module Blazer
end
end
- @columns = {}
+ @first_row = @rows.first || []
+ @column_types = []
if @rows.any?
- @rows.first.each do |key, value|
- @columns[key] =
- case value
+ @columns.each_with_index do |column, i|
+ @column_types << (
+ case @first_row[i]
when Integer
"int"
when Float
@@ -105,20 +106,20 @@ module Blazer
else
"string-ins"
end
+ )
end
end
@filename = @query.name.parameterize if @query
-
- @min_width_types = (@rows.first || {}).select { |k, v| v.is_a?(Time) || v.is_a?(String) || @data_source.smart_columns[k] }.keys
+ @min_width_types = @columns.each_with_index.select { |c, i| @first_row[i].is_a?(Time) || @first_row[i].is_a?(String) || @data_source.smart_columns[c] }
@boom = {}
- @columns.keys.each do |key|
+ @columns.each_with_index do |key, i|
query = @data_source.smart_columns[key]
if query
- values = @rows.map { |r| r[key] }.compact.uniq
- rows, error, cached_at = @data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))
- @boom[key] = Hash[rows.map(&:values).map { |k, v| [k.to_s, v] }]
+ values = @rows.map { |r| r[i] }.compact.uniq
+ columns, rows, error, cached_at = @data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))
+ @boom[key] = Hash[rows.map { |k, v| [k.to_s, v] }]
end
end
@@ -126,15 +127,17 @@ module Blazer
@markers = []
[["latitude", "longitude"], ["lat", "lon"]].each do |keys|
- if (keys - (@rows.first || {}).keys).empty?
+ lat_index = @columns.index(keys.first)
+ lon_index = @columns.index(keys.last)
+ if lat_index && lon_index
@markers =
@rows.select do |r|
- r[keys.first] && r[keys.last]
+ r[lat_index] && r[lon_index]
end.map do |r|
{
- title: r.except(*keys).map{ |k, v| "<strong>#{k}:</strong> #{v}" }.join("<br />").truncate(140),
- latitude: r[keys.first],
- longitude: r[keys.last]
+ title: r.each_with_index.map{ |v, i| i == lat_index || i == lon_index ? nil : "<strong>#{@columns[i]}:</strong> #{v}" }.compact.join("<br />").truncate(140),
+ latitude: r[lat_index],
+ longitude: r[lon_index]
}
end
end
@@ -146,7 +149,7 @@ module Blazer
render layout: false
end
format.csv do
- send_data csv_data(@rows), type: "text/csv; charset=utf-8; header=present", disposition: "attachment; filename=\"#{@query.try(:name).try(:parameterize).presence || 'query'}.csv\""
+ send_data csv_data(@columns, @rows), type: "text/csv; charset=utf-8; header=present", disposition: "attachment; filename=\"#{@query.try(:name).try(:parameterize).presence || 'query'}.csv\""
end
end
end
@@ -181,7 +184,7 @@ module Blazer
end
def tables
- @tables = Blazer.data_sources[params[:data_source]].tables.keys
+ @tables = Blazer.data_sources[params[:data_source]].tables
render partial: "tables", layout: false
end
@@ -224,13 +227,11 @@ module Blazer
params.require(:query).permit(:name, :description, :statement, :data_source)
end
- def csv_data(rows)
+ def csv_data(columns, rows)
CSV.generate do |csv|
- if rows.any?
- csv << rows.first.keys
- end
+ csv << columns
rows.each do |row|
- csv << row.map { |k, v| v.is_a?(Time) ? blazer_time_value(k, v) : v }
+ csv << row.each_with_index.map { |v, i| v.is_a?(Time) ? blazer_time_value(columns[i], v) : v }
end
end
end
diff --git a/app/helpers/blazer/base_helper.rb b/app/helpers/blazer/base_helper.rb
index 6fe7e583..7232aa5a 100644
--- a/app/helpers/blazer/base_helper.rb
+++ b/app/helpers/blazer/base_helper.rb
@@ -17,8 +17,8 @@ module Blazer
end
def blazer_column_types(columns, rows, boom)
- columns.map do |k, _|
- v = (rows.find { |r| r[k] } || {})[k]
+ columns.each_with_index.map do |k, i|
+ v = rows.find { |r| r[i] }[i]
if boom[k]
"string"
elsif v.is_a?(Numeric)
diff --git a/app/views/blazer/queries/run.html.erb b/app/views/blazer/queries/run.html.erb
index 1e9cb290..f64f864b 100644
--- a/app/views/blazer/queries/run.html.erb
+++ b/app/views/blazer/queries/run.html.erb
@@ -24,13 +24,13 @@
<p class="text-muted"><%= pluralize(@rows.size, "row") %></p>
<% end %>
<% if @rows.any? %>
- <% values = @rows.first.values %>
+ <% values = @rows.first %>
<% chart_id = SecureRandom.hex %>
<% column_types = blazer_column_types(@columns, @rows, @boom) %>
<% chart_options = {id: chart_id, min: nil} %>
<% series_library = {} %>
- <% @columns.keys.select { |k| k.downcase == "target" }.each do |key| %>
- <% series_library[key] = {pointStyle: "line", hitRadius: 5, borderColor: "#109618", pointBackgroundColor: "#109618", backgroundColor: "#109618"} %>
+ <% @columns.select { |k| k.downcase == "target" }.each_with_index do |key, i| %>
+ <% series_library[i] = {pointStyle: "line", hitRadius: 5, borderColor: "#109618", pointBackgroundColor: "#109618", backgroundColor: "#109618"} %>
<% end %>
<% if blazer_maps? && @markers.any? %>
<div id="map" style="height: <%= @only_chart ? 300 : 500 %>px;"></div>
@@ -62,32 +62,30 @@
map.fitBounds(featureLayer.getBounds());
</script>
<% elsif values.size >= 2 && column_types.compact == ["time"] + (column_types.compact.size - 1).times.map { "numeric" } %>
- <% time_k = @columns.keys.first %>
- <%= line_chart @columns.keys[1..-1].map{ |k| {name: k, data: @rows.map{|r| [r[time_k], r[k]] }, library: series_library[k]} }, chart_options %>
+ <%= line_chart @columns[1..-1].each_with_index.map{ |k, i| {name: k, data: @rows.map{ |r| [r[0], r[i + 1]] }, library: series_library[i]} }, chart_options %>
<% elsif values.size == 3 && column_types == ["time", "string", "numeric"] %>
- <% keys = @columns.keys %>
- <%= line_chart @rows.group_by { |r| k = keys[1]; v = r[k]; (@boom[k] || {})[v.to_s] || v }.map { |name, v| {name: name, data: v.map { |v2| [v2[keys[0]], v2[keys[2]]] }, library: series_library[name]} }, chart_options %>
+ <%= line_chart @rows.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: name, data: v.map { |v2| [v2[0], v2[2]] }, library: series_library[i]} }, chart_options %>
<% elsif values.size >= 2 && column_types == ["string"] + (values.size - 1).times.map { "numeric" } %>
- <% keys = @columns.keys %>
- <%= column_chart (values.size - 1).times.map { |i| name = @columns.keys[i + 1]; {name: name, data: @rows.first(20).map { |r| [(@boom[keys[0]] || {})[r[keys[0]].to_s] || r[keys[0]], r[keys[i + 1]]] } } }, id: chart_id %>
+ <%= column_chart (values.size - 1).times.map { |i| name = @columns[i + 1]; {name: name, data: @rows.first(20).map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] } } }, id: chart_id %>
<% elsif @only_chart %>
<% if @rows.size == 1 && @rows.first.size == 1 %>
- <p style="font-size: 160px;"><%= blazer_format_value(@rows.first.keys.first, @rows.first.values.first) %></p>
+ <p style="font-size: 160px;"><%= blazer_format_value(@columns.first, @rows.first.first) %></p>
<% else %>
<% @no_chart = true %>
<% end %>
<% end %>
<% unless @only_chart && !@no_chart %>
- <% header_width = 100 / @rows.first.keys.size.to_f %>
+ <% header_width = 100 / @columns.size.to_f %>
<div class="results-container">
- <% if @columns.keys == ["QUERY PLAN"] %>
- <pre><code><%= @rows.map { |r| r["QUERY PLAN"] }.join("\n") %></code></pre>
+ <% if @columns == ["QUERY PLAN"] %>
+ <pre><code><%= @rows.map { |r| r[0] }.join("\n") %></code></pre>
<% else %>
<table class="table results-table" style="margin-bottom: 0;">
<thead>
<tr>
- <% @columns.each do |key, type| %>
+ <% @columns.each_with_index do |key, i| %>
+ <% type = @column_types[i] %>
<th style="width: <%= header_width %>%;" data-sort="<%= type %>">
<div style="min-width: <%= @min_width_types.include?(key) ? 180 : 60 %>px;">
<%= key %>
@@ -99,7 +97,8 @@
<tbody>
<% @rows.each do |row| %>
<tr>
- <% row.each do |k, v| %>
+ <% row.each_with_index do |v, i| %>
+ <% k = @columns[i] %>
<td>
<% if v.is_a?(Time) %>
<% v = blazer_time_value(k, v) %>
diff --git a/lib/blazer.rb b/lib/blazer.rb
index 8d707902..1bafd2b9 100644
--- a/lib/blazer.rb
+++ b/lib/blazer.rb
@@ -72,7 +72,7 @@ module Blazer
ActiveSupport::Notifications.instrument("run_check.blazer", check_id: check.id, query_id: check.query.id, state_was: check.state) do |instrument|
# try 3 times on timeout errors
while tries <= 3
- rows, error, cached_at = data_sources[check.query.data_source].run_statement(check.query.statement, refresh_cache: true)
+ columns, rows, error, cached_at = data_sources[check.query.data_source].run_statement(check.query.statement, refresh_cache: true)
if error == Blazer::TIMEOUT_MESSAGE
Rails.logger.info "[blazer timeout] query=#{check.query.name}"
tries += 1
diff --git a/lib/blazer/data_source.rb b/lib/blazer/data_source.rb
index 9ce2ae9f..c009a3b4 100644
--- a/lib/blazer/data_source.rb
+++ b/lib/blazer/data_source.rb
@@ -58,16 +58,18 @@ module Blazer
end
def run_statement(statement, options = {})
+ columns = nil
rows = nil
error = nil
cached_at = nil
cache_key = self.cache_key(statement) if cache
if cache && !options[:refresh_cache]
value = Blazer.cache.read(cache_key)
- rows, cached_at = Marshal.load(value) if value
+ columns, rows, cached_at = Marshal.load(value) if value
end
unless rows
+ columns = []
rows = []
comment = "blazer"
@@ -95,13 +97,10 @@ module Blazer
end
result = connection_model.connection.select_all("#{statement} /*#{comment}*/")
+ columns = result.columns
cast_method = Rails::VERSION::MAJOR < 5 ? :type_cast : :cast_value
- result.each do |untyped_row|
- row = {}
- untyped_row.each do |k, v|
- row[k] = result.column_types.empty? ? v : result.column_types[k].send(cast_method, v)
- end
- rows << row
+ result.rows.each do |untyped_row|
+ rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| result.column_types[c].send(cast_method, untyped_row[i]) })
end
rescue ActiveRecord::StatementInvalid => e
error = e.message.sub(/.+ERROR: /, "")
@@ -109,10 +108,10 @@ module Blazer
end
end
- Blazer.cache.write(cache_key, Marshal.dump([rows, Time.now]), expires_in: cache.to_f * 60) if !error && cache
+ Blazer.cache.write(cache_key, Marshal.dump([columns, rows, Time.now]), expires_in: cache.to_f * 60) if !error && cache
end
- [rows, error, cached_at]
+ [columns, rows, error, cached_at]
end
def clear_cache(statement)
@@ -120,7 +119,7 @@ module Blazer
end
def cache_key(statement)
- ["blazer", "v2", id, Digest::MD5.hexdigest(statement)].join("/")
+ ["blazer", "v3", id, Digest::MD5.hexdigest(statement)].join("/")
end
def schemas
@@ -129,8 +128,8 @@ module Blazer
end
def tables
- rows, error, cached_at = run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name, column_name, ordinal_position, data_type FROM information_schema.columns WHERE table_schema IN (?)", schemas]))
- Hash[rows.group_by { |r| r["table_name"] }.map { |t, f| [t, f.sort_by { |f| f["ordinal_position"] }.map { |f| f.slice("column_name", "data_type") }] }.sort_by { |t, _f| t }]
+ columns, rows, error, cached_at = run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name, column_name, ordinal_position, data_type FROM information_schema.columns WHERE table_schema IN (?)", schemas]))
+ rows.map(&:first)
end
def postgresql?