Business intelligence made simple
Created result object for cleaner code
Andrew Kane728515ff024d
diff --git a/app/controllers/blazer/dashboards_controller.rb b/app/controllers/blazer/dashboards_controller.rbindex e4aa9263..ea8c1231 100644--- a/app/controllers/blazer/dashboards_controller.rb+++ b/app/controllers/blazer/dashboards_controller.rb@@ -37,9 +37,9 @@ module Blazer@data_sources.each do |data_source|query = data_source.smart_variables[var]if query- columns, rows, error, cached_at = data_source.run_statement(query)- ((@smart_vars[var] ||= []).concat(rows.map { |v| v.reverse })).uniq!- @sql_errors << error if error+ result = data_source.run_statement(query)+ ((@smart_vars[var] ||= []).concat(result.rows.map { |v| v.reverse })).uniq!+ @sql_errors << result.error if result.errorendendenddiff --git a/app/controllers/blazer/queries_controller.rb b/app/controllers/blazer/queries_controller.rbindex 1e952df7..69614945 100644--- a/app/controllers/blazer/queries_controller.rb+++ b/app/controllers/blazer/queries_controller.rb@@ -54,9 +54,9 @@ module Blazer@bind_vars.each do |var|query = data_source.smart_variables[var]if query- columns, rows, error, cached_at = data_source.run_statement(query)- @smart_vars[var] = rows.map { |v| v.reverse }- @sql_errors << error if error+ result = data_source.run_statement(query)+ @smart_vars[var] = result.rows.map { |v| v.reverse }+ @sql_errors << result.error if result.errorendend@@ -110,11 +110,10 @@ module Blazerbreak if result.any? || Time.now - wait_start > 3endelse- result = @data_source.run_main_statement(@statement, options)+ @result = @data_source.run_main_statement(@statement, options)end- if result.any?- @columns, @rows, @error, @cached_at, @just_cached = result+ if @result@data_source.delete_results(@run_id) if @run_idrender_runelse@@ -167,6 +166,12 @@ module Blazerenddef render_run+ @columns = @result.columns+ @rows = @result.rows+ @error = @result.error+ @cached_at = @result.cached_at+ @just_cached = @result.just_cached+@checks = @query ? @query.checks : []@first_row = @rows.first || []@@ -189,7 +194,7 @@ module Blazer@filename = @query.name.parameterize if @query@min_width_types = @columns.each_with_index.select { |c, i| @first_row[i].is_a?(Time) || @first_row[i].is_a?(String) || @data_source.smart_columns[c] }- @boom = Blazer.boom(@columns, @rows, @data_source)+ @boom = @result.boom@linked_columns = @data_source.linked_columnsdiff --git a/app/models/blazer/check.rb b/app/models/blazer/check.rbindex 25c48019..ecdbb3ca 100644--- a/app/models/blazer/check.rb+++ b/app/models/blazer/check.rb@@ -15,7 +15,7 @@ module Blazeremails.to_s.downcase.split(",").map(&:strip)end- def update_state(columns, rows, error, data_source)+ def update_state(result)check_type =if respond_to?(:check_type)self.check_type@@ -25,17 +25,15 @@ module Blazer"bad_data"end- message = error+ message = result.errorself.state =- if error- if error == Blazer::TIMEOUT_MESSAGE- "timed out"- else- "error"- end+ if result.timed_out?+ "timed out"+ elsif result.error+ "error"elsif check_type == "anomaly"- anomaly, message = Blazer.detect_anomaly(columns, rows, data_source)+ anomaly, message = result.detect_anomalyif anomaly.nil?"error"elsif anomaly@@ -43,7 +41,7 @@ module Blazerelse"passing"end- elsif rows.any?+ elsif result.rows.any?check_type == "missing_data" ? "passing" : "failing"elsecheck_type == "missing_data" ? "failing" : "passing"@@ -53,7 +51,7 @@ module Blazerself.message = message if respond_to?(:message=)if respond_to?(:timeouts=)- if state == "timed out"+ if result.timed_out?self.timeouts += 1self.state = "disabled" if timeouts >= 3else@@ -63,7 +61,7 @@ module Blazer# do not notify on creation, except when not passingif (state_was || state != "passing") && state != state_was && emails.present?- Blazer::CheckMailer.state_change(self, state, state_was, rows.size, message).deliver_later+ Blazer::CheckMailer.state_change(self, state, state_was, result.rows.size, message).deliver_laterendsave! if changed?enddiff --git a/app/views/blazer/queries/run.html.erb b/app/views/blazer/queries/run.html.erbindex a2d44b77..38c7fb5f 100644--- a/app/views/blazer/queries/run.html.erb+++ b/app/views/blazer/queries/run.html.erb@@ -38,8 +38,8 @@<% if @rows.any? %><% values = @rows.first %><% chart_id = SecureRandom.hex %>- <% column_types = Blazer.column_types(@columns, @rows, @boom) %>- <% chart_type = Blazer.chart_type(column_types) %>+ <% column_types = @result.column_types %>+ <% chart_type = @result.chart_type %><% chart_options = {id: chart_id, min: nil} %><% series_library = {} %><% target_index = @columns.index { |k| k.downcase == "target" } %>@@ -81,7 +81,7 @@<%= line_chart @rows.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: name, data: v.map { |v2| [v2[0], v2[2]] }, library: series_library[i]} }, chart_options %><% elsif chart_type == "bar" %><%= column_chart (values.size - 1).times.map { |i| name = @columns[i + 1]; {name: name, data: @rows.first(20).map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] } } }, id: chart_id %>- <% elsif values.size == 3 && column_types == ["string", "string", "numeric"] %>+ <% elsif chart_type == "bar2" %><% first_20 = @rows.group_by { |r| r[0] }.values.first(20).flatten(1) %><% labels = first_20.map { |r| r[0] }.uniq %><% series = first_20.map { |r| r[1] }.uniq %>diff --git a/lib/blazer.rb b/lib/blazer.rbindex 81b21e29..683d42ef 100644--- a/lib/blazer.rb+++ b/lib/blazer.rb@@ -1,10 +1,11 @@require "csv"require "yaml"require "chartkick"+require "safely/core"require "blazer/version"require "blazer/data_source"+require "blazer/result"require "blazer/engine"-require "safely/core"module Blazerclass Error < StandardError; end@@ -90,12 +91,12 @@ module BlazerBlazer.transform_statement.call(data_source, statement) if Blazer.transform_statementwhile tries <= 3- columns, rows, error, cached_at = data_source.run_statement(statement, refresh_cache: true, check: check, query: check.query)- if error == Blazer::TIMEOUT_MESSAGE+ result = data_source.run_statement(statement, refresh_cache: true, check: check, query: check.query)+ if result.timed_out?Rails.logger.info "[blazer timeout] query=#{check.query.name}"tries += 1sleep(10)- elsif error.to_s.start_with?("PG::ConnectionBad")+ elsif result.error.to_s.start_with?("PG::ConnectionBad")data_source.reconnectRails.logger.info "[blazer reconnect] query=#{check.query.name}"tries += 1@@ -104,15 +105,15 @@ module Blazerbreakendend- check.update_state(columns, rows, error, data_source)+ check.update_state(result)# TODO use proper logfmt- Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{rows.try(:size)} error=#{error}"+ Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{result.rows.try(:size)} error=#{result.error}"instrument[:statement] = statementinstrument[:data_source] = data_sourceinstrument[:state] = check.state- instrument[:rows] = rows.try(:size)- instrument[:error] = error+ instrument[:rows] = result.rows.try(:size)+ instrument[:error] = result.errorinstrument[:tries] = triesendend@@ -129,123 +130,4 @@ module BlazerBlazer::CheckMailer.failing_checks(email, checks).deliver_laterendend-- def self.column_types(columns, rows, boom = {})- columns.each_with_index.map do |k, i|- v = (rows.find { |r| r[i] } || {})[i]- if boom[k]- "string"- elsif v.is_a?(Numeric)- "numeric"- elsif v.is_a?(Time) || v.is_a?(Date)- "time"- elsif v.nil?- nil- else- "string"- end- end- end-- def self.chart_type(column_types)- if column_types.compact.size >= 2 && column_types.compact == ["time"] + (column_types.compact.size - 1).times.map { "numeric" }- "line"- elsif column_types == ["time", "string", "numeric"]- "line2"- elsif column_types.compact.size >= 2 && column_types == ["string"] + (column_types.compact.size - 1).times.map { "numeric" }- "bar"- end- end-- def self.detect_anomaly(columns, rows, data_source)- anomaly = nil- message = nil-- if rows.empty?- message = "No data"- else- boom = self.boom(columns, rows, data_source)- chart_type = self.chart_type(column_types(columns, rows, boom))- if chart_type == "line" || chart_type == "line2"- series = []-- if chart_type == "line"- columns[1..-1].each_with_index.each do |k, i|- series << {name: k, data: rows.map{ |r| [r[0], r[i + 1]] }}- end- else- rows.group_by { |r| v = r[1]; (boom[columns[1]] || {})[v.to_s] || v }.each_with_index.map do |(name, v), i|- series << {name: name, data: v.map { |v2| [v2[0], v2[2]] }}- end- end-- current_series = nil- begin- anomalies = []- series.each do |s|- current_series = s[:name]- anomalies << s[:name] if anomaly?(s[:data])- end- anomaly = anomalies.any?- if anomaly- if anomalies.size == 1- message = "Anomaly detected in #{anomalies.first}"- else- message = "Anomalies detected in #{anomalies.to_sentence}"- end- else- message = "No anomalies detected"- end- rescue => e- message = "#{current_series}: #{e.message}"- end- else- message = "Bad format"- end- end-- [anomaly, message]- end-- def self.anomaly?(series)- series = series.reject { |v| v[0].nil? }.sort_by { |v| v[0] }-- csv_str =- CSV.generate do |csv|- csv << ["timestamp", "count"]- series.each do |row|- csv << row- end- end-- timestamps = []- r_script = %x[which Rscript].chomp- raise "R not found" if r_script.empty?- output = %x[#{r_script} --vanilla #{File.expand_path("../blazer/detect_anomalies.R", __FILE__)} #{Shellwords.escape(csv_str)}]- if output.empty?- raise "Unknown R error"- end-- rows = CSV.parse(output, headers: true)- error = rows.first && rows.first["x"]- raise error if error-- rows.each do |row|- timestamps << Time.parse(row["timestamp"])- end- timestamps.include?(series.last[0].to_time)- end-- def self.boom(columns, rows, data_source)- boom = {}- columns.each_with_index do |key, i|- query = data_source.smart_columns[key]- if query- values = rows.map { |r| r[i] }.compact.uniq- columns, rows2, error, cached_at = data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))- boom[key] = Hash[rows2.map { |k, v| [k.to_s, v] }]- end- end- boom- endenddiff --git a/lib/blazer/data_source.rb b/lib/blazer/data_source.rbindex 18ed5e4a..6710e940 100644--- a/lib/blazer/data_source.rb+++ b/lib/blazer/data_source.rb@@ -110,32 +110,34 @@ module Blazerendstart_time = Time.now- columns, rows, error, cached_at, just_cached = run_statement(statement, options.merge(with_just_cached: true))+ result = run_statement(statement, options.merge(with_just_cached: true))duration = Time.now - start_timeif Blazer.auditaudit.duration = duration if audit.respond_to?(:duration=)- audit.error = error if audit.respond_to?(:error=)- audit.timed_out = error == Blazer::TIMEOUT_MESSAGE if audit.respond_to?(:timed_out=)- audit.cached = cached_at.present? if audit.respond_to?(:cached=)- if !cached_at && duration >= 10+ audit.error = result.error if audit.respond_to?(:error=)+ audit.timed_out = result.timed_out? if audit.respond_to?(:timed_out=)+ audit.cached = result.cached? if audit.respond_to?(:cached=)+ if !result.cached? && duration >= 10audit.cost = cost(statement) if audit.respond_to?(:cost=)endaudit.save! if audit.changed?end- if query && error != Blazer::TIMEOUT_MESSAGE+ if query && !result.timed_out?query.checks.each do |check|- check.update_state(columns, rows, error, self)+ check.update_state(result)endend- [columns, rows, error, cached_at, just_cached]+ resultenddef read_cache(cache_key)value = Blazer.cache.read(cache_key)- Marshal.load(value) if value+ if value+ Blazer::Result.new(*Marshal.load(value))+ endenddef run_results(run_id)@@ -155,7 +157,7 @@ module Blazerrun_id = options[:run_id]cache_key = statement_cache_key(statement) if cache_mode != "off"if cache_mode != "off" && !options[:refresh_cache]- columns, rows, error, cached_at = read_cache(cache_key)+ result = read_cache(cache_key)endunless rows@@ -173,12 +175,10 @@ module Blazerif options[:check]comment << ",check_id:#{options[:check].id},check_emails:#{options[:check].emails}"end- columns, rows, error, just_cached = run_statement_helper(statement, comment, options[:run_id])+ result = run_statement_helper(statement, comment, options[:run_id])end- output = [columns, rows, error, cached_at]- output << just_cached if options[:with_just_cached]- output+ resultenddef clear_cache(statement)@@ -203,8 +203,8 @@ module Blazerenddef tables- columns, rows, error, cached_at = run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name FROM information_schema.tables WHERE table_schema IN (?) ORDER BY table_name", schemas]))- rows.map(&:first)+ result = run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name FROM information_schema.tables WHERE table_schema IN (?) ORDER BY table_name", schemas]))+ result.rows.map(&:first)enddef postgresql?@@ -279,7 +279,7 @@ module BlazerBlazer.cache.write(run_cache_key(run_id), cache_data, expires_in: 30.seconds)end- [columns, rows, error, cache && !cache_data.nil?]+ Blazer::Result.new(self, columns, rows, error, nil, cache && !cache_data.nil?)enddef adapter_namediff --git a/lib/blazer/result.rb b/lib/blazer/result.rbnew file mode 100644index 00000000..048b0e16--- /dev/null+++ b/lib/blazer/result.rb@@ -0,0 +1,147 @@+module Blazer+ class Result+ attr_reader :data_source, :columns, :rows, :error, :cached_at, :just_cached++ def initialize(data_source, columns, rows, error, cached_at, just_cached)+ @data_source = data_source+ @columns = columns+ @rows = rows+ @error = error+ @cached_at = cached_at+ @just_cached = just_cached+ end++ def timed_out?+ error == Blazer::TIMEOUT_MESSAGE+ end++ def cached?+ cached_at.present?+ end++ def boom+ @boom ||= begin+ boom = {}+ columns.each_with_index do |key, i|+ query = data_source.smart_columns[key]+ if query+ values = rows.map { |r| r[i] }.compact.uniq+ result = data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))+ boom[key] = Hash[result.rows.map { |k, v| [k.to_s, v] }]+ end+ end+ boom+ end+ end++ def column_types+ @column_types ||= begin+ columns.each_with_index.map do |k, i|+ v = (rows.find { |r| r[i] } || {})[i]+ if boom[k]+ "string"+ elsif v.is_a?(Numeric)+ "numeric"+ elsif v.is_a?(Time) || v.is_a?(Date)+ "time"+ elsif v.nil?+ nil+ else+ "string"+ end+ end+ end+ end++ def chart_type+ @chart_type ||= begin+ if column_types.compact.size >= 2 && column_types.compact == ["time"] + (column_types.compact.size - 1).times.map { "numeric" }+ "line"+ elsif column_types == ["time", "string", "numeric"]+ "line2"+ elsif column_types.compact.size >= 2 && column_types == ["string"] + (column_types.compact.size - 1).times.map { "numeric" }+ "bar"+ elsif column_types == ["string", "string", "numeric"]+ "bar2"+ end+ end+ end++ def detect_anomaly+ anomaly = nil+ message = nil++ if rows.empty?+ message = "No data"+ else+ if chart_type == "line" || chart_type == "line2"+ series = []++ if chart_type == "line"+ columns[1..-1].each_with_index.each do |k, i|+ series << {name: k, data: rows.map{ |r| [r[0], r[i + 1]] }}+ end+ else+ rows.group_by { |r| v = r[1]; (boom[columns[1]] || {})[v.to_s] || v }.each_with_index.map do |(name, v), i|+ series << {name: name, data: v.map { |v2| [v2[0], v2[2]] }}+ end+ end++ current_series = nil+ begin+ anomalies = []+ series.each do |s|+ current_series = s[:name]+ anomalies << s[:name] if anomaly?(s[:data])+ end+ anomaly = anomalies.any?+ if anomaly+ if anomalies.size == 1+ message = "Anomaly detected in #{anomalies.first}"+ else+ message = "Anomalies detected in #{anomalies.to_sentence}"+ end+ else+ message = "No anomalies detected"+ end+ rescue => e+ message = "#{current_series}: #{e.message}"+ end+ else+ message = "Bad format"+ end+ end++ [anomaly, message]+ end++ def anomaly?(series)+ series = series.reject { |v| v[0].nil? }.sort_by { |v| v[0] }++ csv_str =+ CSV.generate do |csv|+ csv << ["timestamp", "count"]+ series.each do |row|+ csv << row+ end+ end++ timestamps = []+ r_script = %x[which Rscript].chomp+ raise "R not found" if r_script.empty?+ output = %x[#{r_script} --vanilla #{File.expand_path("../blazer/detect_anomalies.R", __FILE__)} #{Shellwords.escape(csv_str)}]+ if output.empty?+ raise "Unknown R error"+ end++ rows = CSV.parse(output, headers: true)+ error = rows.first && rows.first["x"]+ raise error if error++ rows.each do |row|+ timestamps << Time.parse(row["timestamp"])+ end+ timestamps.include?(series.last[0].to_time)+ end+ end+end