Business intelligence made simple
Added cohorts
Andrew Kanec4c3e89d5a6d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex d69d16a8..14b42b2c 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,5 +1,6 @@-## 2.3.2 (unreleased)+## 2.4.0 (unreleased)+- Added cohorts- Fixed broken routes for some applications## 2.3.1 (2020-11-23)diff --git a/README.md b/README.mdindex 741b10a4..6905eb63 100644--- a/README.md+++ b/README.md@@ -25,6 +25,7 @@ Blazer is also available as a [Docker image](https://github.com/ankane/blazer-do- [Charts](#charts)- [Dashboards](#dashboards)- [Checks](#checks)+- [Cohorts](#cohorts)- [Anomaly Detection](#anomaly-detection)- [Forecasting](#forecasting)- [Uploads](#uploads)@@ -388,6 +389,31 @@ SELECT * FROM ratings WHERE user_id IS NULL /* all ratings should have a user */Then create check with optional emails if you want to be notified. Emails are sent when a check starts failing, and when it starts passing again.+## Cohorts++Create a cohort analysis from a simple SQL query. [Example](https://blazer.dokkuapp.com/queries/19-cohort-analysis-from-first-order)++Create a query with the comment `/* cohort analysis */`. The result should have columns named `user_id` and `conversion_time` and optionally `cohort_time`.++You can generate cohorts from the first conversion time:++```sql+/* cohort analysis */+SELECT user_id, created_at AS conversion_time FROM orders+```++(the first conversion isn’t counted in the first time period with this format)++Or from another time, like sign up:++```sql+/* cohort analysis */+SELECT users.id AS user_id, orders.created_at AS conversion_time, users.created_at AS cohort_time+FROM users LEFT JOIN orders ON orders.user_id = users.id+```++This feature requires PostgreSQL.+## Anomaly DetectionBlazer supports two different approaches to anomaly detection.diff --git a/app/assets/stylesheets/blazer/application.css b/app/assets/stylesheets/blazer/application.cssindex 4cb57fe8..50fb25d8 100644--- a/app/assets/stylesheets/blazer/application.css+++ b/app/assets/stylesheets/blazer/application.css@@ -16,6 +16,10 @@ body {padding-bottom: 15px;}+table.results-table {+ margin-bottom: 0;+}+.results-table th {cursor: pointer;}diff --git a/app/controllers/blazer/base_controller.rb b/app/controllers/blazer/base_controller.rbindex 3fe86579..49faa611 100644--- a/app/controllers/blazer/base_controller.rb+++ b/app/controllers/blazer/base_controller.rb@@ -67,6 +67,12 @@ module Blazerendend+ def add_cohort_analysis_vars+ @bind_vars << "cohort_period" unless @bind_vars.include?("cohort_period")+ @smart_vars["cohort_period"] = ["day", "week", "month"]+ params[:cohort_period] ||= "week"+ end+def parse_smart_variables(var, data_source)smart_var_data_source =([data_source] + Array(data_source.settings["inherit_smart_settings"]).map { |ds| Blazer.data_sources[ds] }).find { |ds| ds.smart_variables[var] }diff --git a/app/controllers/blazer/dashboards_controller.rb b/app/controllers/blazer/dashboards_controller.rbindex a1478c23..78bf6b48 100644--- a/app/controllers/blazer/dashboards_controller.rb+++ b/app/controllers/blazer/dashboards_controller.rb@@ -39,6 +39,8 @@ module Blazer@sql_errors << error if errorendend++ add_cohort_analysis_vars if @queries.any?(&:cohort_analysis?)enddef editdiff --git a/app/controllers/blazer/queries_controller.rb b/app/controllers/blazer/queries_controller.rbindex 82e2daf6..d71b33a9 100644--- a/app/controllers/blazer/queries_controller.rb+++ b/app/controllers/blazer/queries_controller.rb@@ -74,6 +74,8 @@ module Blazer@query.update!(status: "active") if @query.try(:status) == "archived"Blazer.transform_statement.call(data_source, @statement) if Blazer.transform_statement++ add_cohort_analysis_vars if @query.cohort_analysis?enddef edit@@ -81,6 +83,8 @@ module Blazerdef run@statement = params[:statement]+ # before process_vars+ @cohort_analysis = Query.new(statement: @statement).cohort_analysis?data_source = params[:data_source]process_vars(@statement, data_source)@only_chart = params[:only_chart]@@ -89,6 +93,8 @@ module Blazerdata_source = @query.data_source if @query && @query.data_source@data_source = Blazer.data_sources[data_source]+ run_cohort_analysis if @cohort_analysis+# ensure viewableif !(@query || Query.new(data_source: @data_source.id)).viewable?(blazer_user)render_forbidden@@ -164,6 +170,7 @@ module Blazer@statement = @query.statement.dupprocess_vars(@statement, @query.data_source)Blazer.transform_statement.call(data_source, @statement) if Blazer.transform_statement+ @statement = cohort_analysis_statement(data_source, @statement) if @query.cohort_analysis?data_source.clear_cache(@statement)redirect_to query_path(@query, variable_params(@query))end@@ -267,6 +274,8 @@ module Blazerendend+ render_cohort_analysis if @cohort_analysis && !@error+respond_to do |format|format.html dorender layout: false@@ -351,5 +360,104 @@ module Blazerdef blazer_run_idparams[:run_id].to_s.gsub(/[^a-z0-9\-]/i, "")end++ def run_cohort_analysis+ unless @data_source.supports_cohort_analysis?+ @cohort_error = "This data source does not support cohort analysis"+ end++ @show_cohort_rows = !params[:query_id] || @cohort_error++ unless @show_cohort_rows+ @statement = cohort_analysis_statement(@data_source, @statement)+ end+ end++ def cohort_analysis_statement(data_source, statement)+ @cohort_period = params["cohort_period"] || "week"+ @cohort_period = "week" unless ["day", "week", "month"].include?(@cohort_period)++ # for now+ @conversion_period = @cohort_period+ @cohort_days =+ case @cohort_period+ when "day"+ 1+ when "week"+ 7+ when "month"+ 30+ end++ data_source.cohort_analysis_statement(statement, period: @cohort_period, days: @cohort_days)+ end++ def render_cohort_analysis+ if @show_cohort_rows+ @cohort_analysis = false++ @row_limit = 1000++ # check results+ unless @cohort_error+ # check names+ expected_columns = ["user_id", "conversion_time"]+ missing_columns = expected_columns - @result.columns+ if missing_columns.any?+ @cohort_error = "Expected user_id and conversion_time columns"+ end++ # check types (user_id can be any type)+ unless @cohort_error+ column_types = @result.columns.zip(@result.column_types).to_h++ if !column_types["cohort_time"].in?(["time", nil])+ @cohort_error = "cohort_time must be time column"+ elsif !column_types["conversion_time"].in?(["time", nil])+ @cohort_error = "conversion_time must be time column"+ end+ end+ end+ else+ @today = Blazer.time_zone.today+ @min_cohort_date, @max_cohort_date = @result.rows.map { |r| r[0] }.minmax+ @buckets = {}+ @rows.each do |r|+ @buckets[[r[0], r[1]]] = r[2]+ end++ @cohort_dates = []+ current_date = @max_cohort_date+ while current_date && current_date >= @min_cohort_date+ @cohort_dates << current_date+ current_date =+ case @cohort_period+ when "day"+ current_date - 1+ when "week"+ current_date - 7+ else+ current_date.prev_month+ end+ end++ num_cols = @cohort_dates.size+ @columns = ["Cohort", "Users"] + num_cols.times.map { |i| "#{@conversion_period.titleize} #{i + 1}" }+ rows = []+ date_format = @cohort_period == "month" ? "%b %Y" : "%b %-e, %Y"+ @cohort_dates.each do |date|+ row = [date.strftime(date_format), @buckets[[date, 0]] || 0]++ num_cols.times do |i|+ if @today >= date + (@cohort_days * i)+ row << (@buckets[[date, i + 1]] || 0)+ end+ end++ rows << row+ end+ @rows = rows+ end+ endendenddiff --git a/app/models/blazer/query.rb b/app/models/blazer/query.rbindex 0b392092..9f6f1690 100644--- a/app/models/blazer/query.rb+++ b/app/models/blazer/query.rb@@ -35,7 +35,13 @@ module Blazerenddef variables- Blazer.extract_vars(statement)+ variables = Blazer.extract_vars(statement)+ variables += ["cohort_period"] if cohort_analysis?+ variables+ end++ def cohort_analysis?+ /\/\*\s*cohort analysis\s*\*\//i.match?(statement)endendenddiff --git a/app/views/blazer/dashboards/show.html.erb b/app/views/blazer/dashboards/show.html.erbindex d35a5199..4878fb0b 100644--- a/app/views/blazer/dashboards/show.html.erb+++ b/app/views/blazer/dashboards/show.html.erb@@ -39,7 +39,7 @@</div></div><script>- <%= blazer_js_var "data", {statement: @statements[i], query_id: query.id, data_source: query.data_source, only_chart: true} %>+ <%= blazer_js_var "data", {statement: @statements[i], query_id: query.id, data_source: query.data_source, only_chart: true, cohort_period: params[:cohort_period]} %>runQuery(data, function (data) {$("#chart-<%= i %>").html(data)diff --git a/app/views/blazer/queries/_caching.html.erb b/app/views/blazer/queries/_caching.html.erbnew file mode 100644index 00000000..31e820e2--- /dev/null+++ b/app/views/blazer/queries/_caching.html.erb@@ -0,0 +1,16 @@+<% if @cached_at || @just_cached %>+ <p class="text-muted" style="float: right;">+ <% if @cached_at %>+ Cached <%= time_ago_in_words(@cached_at, include_seconds: true) %> ago+ <% elsif params[:query_id] %>+ Cached just now+ <% if @data_source.cache_mode == "slow" %>+ (over <%= "%g" % @data_source.cache_slow_threshold %>s)+ <% end %>+ <% end %>++ <% if @query && params[:query_id] %>+ <%= link_to "Refresh", refresh_query_path(@query, variable_params(@query)), method: :post %>+ <% end %>+ </p>+<% end %>diff --git a/app/views/blazer/queries/_cohorts.html.erb b/app/views/blazer/queries/_cohorts.html.erbnew file mode 100644index 00000000..ff434475--- /dev/null+++ b/app/views/blazer/queries/_cohorts.html.erb@@ -0,0 +1,48 @@+<% unless @only_chart %>+ <%= render partial: "caching" %>+ <p class="text-muted" style="margin-bottom: 10px;">+ <%= pluralize(@rows.size, "cohort") %>+ </p>+<% end %>+<% if @rows.any? %>+ <div class="results-container">+ <table class="table results-table">+ <thead>+ <tr>+ <th style="min-width: 100px;">Cohort</th>+ <% 12.times do |i| %>+ <th style="width: 7.5%; text-align: right;"><%= @conversion_period.titleize %> <%= i + 1 %></th>+ <% end %>+ </tr>+ </thead>+ <tbody>+ <% @rows.each do |row| %>+ <tr>+ <td>+ <%= row[0] %>+ <div style="font-size: 12px; color: #999;"><%= row[1] == 1 ? "1 user" : "#{number_with_delimiter(row[1])} users" %></div>+ </td>+ <% 12.times do |i| %>+ <td style="text-align: right;">+ <% num = row[i + 2] %>+ <% if num %>+ <% denom = row[1] %>+ <% if denom > 0 %>+ <%= (100.0 * num / denom).round %>%+ <% else %>+ -+ <% end %>+ <div style="font-size: 12px; color: #999;"><%= number_with_delimiter(num) %></div>+ <% else %>+ -+ <% end %>+ </td>+ <% end %>+ </tr>+ <% end %>+ </tbody>+ </table>+ </div>+<% elsif @only_chart %>+ <p class="text-muted">No cohorts</p>+<% end %>diff --git a/app/views/blazer/queries/docs.html.erb b/app/views/blazer/queries/docs.html.erbindex 503c46a2..ba5ea92b 100644--- a/app/views/blazer/queries/docs.html.erb+++ b/app/views/blazer/queries/docs.html.erb@@ -129,3 +129,9 @@</table><p>Use the column name <code>target</code> to draw a line for goals.</p>++<% if @data_source.supports_cohort_analysis? %>+ <h2>Cohort Analysis</h2>++ <p>Create a query with the comment <code>/* cohort analysis */</code>. The result should have columns named <code>user_id</code> and <code>conversion_time</code> and optionally <code>cohort_time</code>.</p>+<% end %>diff --git a/app/views/blazer/queries/run.html.erb b/app/views/blazer/queries/run.html.erbindex 824f2f43..f6c571b2 100644--- a/app/views/blazer/queries/run.html.erb+++ b/app/views/blazer/queries/run.html.erb@@ -6,25 +6,20 @@<% else %><div class="alert alert-info">Can’t preview queries with variables...yet!</div><% end %>+<% elsif @cohort_analysis %>+ <% if @cohort_error %>+ <div class="alert alert-info"><%= @cohort_error %></div>+ <% else %>+ <%= render partial: "cohorts" %>+ <% end %><% else %><% unless @only_chart %>- <% if @cached_at || @just_cached %>- <p class="text-muted" style="float: right;">- <% if @cached_at %>- Cached <%= time_ago_in_words(@cached_at, include_seconds: true) %> ago- <% elsif params[:query_id] %>- Cached just now- <% if @data_source.cache_mode == "slow" %>- (over <%= "%g" % @data_source.cache_slow_threshold %>s)- <% end %>- <% end %>-- <% if @query && params[:query_id] %>- <%= link_to "Refresh", refresh_query_path(@query, variable_params(@query)), method: :post %>- <% end %>- </p>- <% end %>+ <%= render partial: "caching" %><p class="text-muted" style="margin-bottom: 10px;">+ <% if @row_limit && @rows.size > @row_limit %>+ First+ <% @rows = @rows.first(@row_limit) %>+ <% end %><%= pluralize(@rows.size, "row") %><% @checks.select(&:state).each do |check| %>@@ -43,6 +38,9 @@<% if @forecast_error %><div class="alert alert-danger"><%= @forecast_error %></div><% end %>+ <% if @cohort_error %>+ <div class="alert alert-info"><%= @cohort_error %></div>+ <% end %><% if @rows.any? %><% values = @rows.first %><% chart_id = SecureRandom.hex %>@@ -147,7 +145,7 @@<% elsif @columns == ["PLAN"] && @data_source.adapter == "druid" %><pre><code><%= @rows[0][0] %></code></pre><% else %>- <table class="table results-table" style="margin-bottom: 0;">+ <table class="table results-table"><thead><tr><% @columns.each_with_index do |key, i| %>diff --git a/app/views/blazer/queries/show.html.erb b/app/views/blazer/queries/show.html.erbindex afe42c31..c04a59bc 100644--- a/app/views/blazer/queries/show.html.erb+++ b/app/views/blazer/queries/show.html.erb@@ -14,7 +14,7 @@<%= link_to "Fork", new_query_path(variable_params(@query).merge(fork_query_id: @query.id, data_source: @query.data_source, name: @query.name)), class: "btn btn-info" %><% if !@error && @success %>- <%= button_to "Download", run_queries_path(query_id: @query.id, format: "csv", forecast: params[:forecast]), params: {statement: @statement}, class: "btn btn-primary" %>+ <%= button_to "Download", run_queries_path(query_id: @query.id, format: "csv", forecast: params[:forecast], cohort_period: params[:cohort_period]), params: {statement: @statement}, class: "btn btn-primary" %><% end %></div></div>diff --git a/lib/blazer/adapters/base_adapter.rb b/lib/blazer/adapters/base_adapter.rbindex e9b78416..631c943e 100644--- a/lib/blazer/adapters/base_adapter.rb+++ b/lib/blazer/adapters/base_adapter.rb@@ -43,6 +43,14 @@ module Blazertrue # optionalend+ def supports_cohort_analysis?+ false # optional+ end++ def cohort_analysis_statement(statement, period:, days:)+ # optional+ end+protecteddef settingsdiff --git a/lib/blazer/adapters/sql_adapter.rb b/lib/blazer/adapters/sql_adapter.rbindex ba26ef97..f5bd73a1 100644--- a/lib/blazer/adapters/sql_adapter.rb+++ b/lib/blazer/adapters/sql_adapter.rb@@ -122,6 +122,47 @@ module Blazer!%w[CREATE ALTER UPDATE INSERT DELETE].include?(statement.split.first.to_s.upcase)end+ def supports_cohort_analysis?+ postgresql?+ end++ # TODO treat date columns as already in time zone+ def cohort_analysis_statement(statement, period:, days:)+ raise "Cohort analysis not supported" unless supports_cohort_analysis?++ cohort_column = statement =~ /\bcohort_time\b/ ? "cohort_time" : "conversion_time"++ # WITH not an optimization fence in Postgres 12++ statement = <<~SQL+ WITH query AS (+ #{statement}+ ),+ cohorts AS (+ SELECT user_id, MIN(#{cohort_column}) AS cohort_time FROM query+ WHERE user_id IS NOT NULL AND #{cohort_column} IS NOT NULL+ GROUP BY 1+ )+ SELECT+ date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date AS period,+ 0 AS bucket,+ COUNT(DISTINCT cohorts.user_id)+ FROM cohorts GROUP BY 1+ UNION ALL+ SELECT+ date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date AS period,+ CEIL(EXTRACT(EPOCH FROM query.conversion_time - cohorts.cohort_time) / ?)::int AS bucket,+ COUNT(DISTINCT query.user_id)+ FROM cohorts INNER JOIN query ON query.user_id = cohorts.user_id+ WHERE query.conversion_time IS NOT NULL+ AND query.conversion_time >= cohorts.cohort_time+ #{cohort_column == "conversion_time" ? "AND query.conversion_time != cohorts.cohort_time" : ""}+ GROUP BY 1, 2+ SQL+ tzname = Blazer.time_zone.tzinfo.name+ params = [statement, period, tzname, period, tzname, days.to_i * 86400]+ connection_model.send(:sanitize_sql_array, params)+ end+protecteddef select_all(statement, params = [])diff --git a/lib/blazer/data_source.rb b/lib/blazer/data_source.rbindex 8842d341..8ad0fe2d 100644--- a/lib/blazer/data_source.rb+++ b/lib/blazer/data_source.rb@@ -6,7 +6,7 @@ module Blazerattr_reader :id, :settings- def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel+ def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel, :supports_cohort_analysis?, :cohort_analysis_statementdef initialize(id, settings)@id = id