Skip to content

Business intelligence made simple

Added cohorts

Andrew Kanec4c3e89d5a6d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d69d16a8..14b42b2c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,6 @@
-## 2.3.2 (unreleased)
+## 2.4.0 (unreleased)
+- Added cohorts
- Fixed broken routes for some applications
## 2.3.1 (2020-11-23)
diff --git a/README.md b/README.md
index 741b10a4..6905eb63 100644
--- a/README.md
+++ b/README.md
@@ -25,6 +25,7 @@ Blazer is also available as a [Docker image](https://github.com/ankane/blazer-do
- [Charts](#charts)
- [Dashboards](#dashboards)
- [Checks](#checks)
+- [Cohorts](#cohorts)
- [Anomaly Detection](#anomaly-detection)
- [Forecasting](#forecasting)
- [Uploads](#uploads)
@@ -388,6 +389,31 @@ SELECT * FROM ratings WHERE user_id IS NULL /* all ratings should have a user */
Then create check with optional emails if you want to be notified. Emails are sent when a check starts failing, and when it starts passing again.
+## Cohorts
+
+Create a cohort analysis from a simple SQL query. [Example](https://blazer.dokkuapp.com/queries/19-cohort-analysis-from-first-order)
+
+Create a query with the comment `/* cohort analysis */`. The result should have columns named `user_id` and `conversion_time` and optionally `cohort_time`.
+
+You can generate cohorts from the first conversion time:
+
+```sql
+/* cohort analysis */
+SELECT user_id, created_at AS conversion_time FROM orders
+```
+
+(the first conversion isn’t counted in the first time period with this format)
+
+Or from another time, like sign up:
+
+```sql
+/* cohort analysis */
+SELECT users.id AS user_id, orders.created_at AS conversion_time, users.created_at AS cohort_time
+FROM users LEFT JOIN orders ON orders.user_id = users.id
+```
+
+This feature requires PostgreSQL.
+
## Anomaly Detection
Blazer supports two different approaches to anomaly detection.
diff --git a/app/assets/stylesheets/blazer/application.css b/app/assets/stylesheets/blazer/application.css
index 4cb57fe8..50fb25d8 100644
--- a/app/assets/stylesheets/blazer/application.css
+++ b/app/assets/stylesheets/blazer/application.css
@@ -16,6 +16,10 @@ body {
padding-bottom: 15px;
}
+table.results-table {
+ margin-bottom: 0;
+}
+
.results-table th {
cursor: pointer;
}
diff --git a/app/controllers/blazer/base_controller.rb b/app/controllers/blazer/base_controller.rb
index 3fe86579..49faa611 100644
--- a/app/controllers/blazer/base_controller.rb
+++ b/app/controllers/blazer/base_controller.rb
@@ -67,6 +67,12 @@ module Blazer
end
end
+ def add_cohort_analysis_vars
+ @bind_vars << "cohort_period" unless @bind_vars.include?("cohort_period")
+ @smart_vars["cohort_period"] = ["day", "week", "month"]
+ params[:cohort_period] ||= "week"
+ end
+
def parse_smart_variables(var, data_source)
smart_var_data_source =
([data_source] + Array(data_source.settings["inherit_smart_settings"]).map { |ds| Blazer.data_sources[ds] }).find { |ds| ds.smart_variables[var] }
diff --git a/app/controllers/blazer/dashboards_controller.rb b/app/controllers/blazer/dashboards_controller.rb
index a1478c23..78bf6b48 100644
--- a/app/controllers/blazer/dashboards_controller.rb
+++ b/app/controllers/blazer/dashboards_controller.rb
@@ -39,6 +39,8 @@ module Blazer
@sql_errors << error if error
end
end
+
+ add_cohort_analysis_vars if @queries.any?(&:cohort_analysis?)
end
def edit
diff --git a/app/controllers/blazer/queries_controller.rb b/app/controllers/blazer/queries_controller.rb
index 82e2daf6..d71b33a9 100644
--- a/app/controllers/blazer/queries_controller.rb
+++ b/app/controllers/blazer/queries_controller.rb
@@ -74,6 +74,8 @@ module Blazer
@query.update!(status: "active") if @query.try(:status) == "archived"
Blazer.transform_statement.call(data_source, @statement) if Blazer.transform_statement
+
+ add_cohort_analysis_vars if @query.cohort_analysis?
end
def edit
@@ -81,6 +83,8 @@ module Blazer
def run
@statement = params[:statement]
+ # before process_vars
+ @cohort_analysis = Query.new(statement: @statement).cohort_analysis?
data_source = params[:data_source]
process_vars(@statement, data_source)
@only_chart = params[:only_chart]
@@ -89,6 +93,8 @@ module Blazer
data_source = @query.data_source if @query && @query.data_source
@data_source = Blazer.data_sources[data_source]
+ run_cohort_analysis if @cohort_analysis
+
# ensure viewable
if !(@query || Query.new(data_source: @data_source.id)).viewable?(blazer_user)
render_forbidden
@@ -164,6 +170,7 @@ module Blazer
@statement = @query.statement.dup
process_vars(@statement, @query.data_source)
Blazer.transform_statement.call(data_source, @statement) if Blazer.transform_statement
+ @statement = cohort_analysis_statement(data_source, @statement) if @query.cohort_analysis?
data_source.clear_cache(@statement)
redirect_to query_path(@query, variable_params(@query))
end
@@ -267,6 +274,8 @@ module Blazer
end
end
+ render_cohort_analysis if @cohort_analysis && !@error
+
respond_to do |format|
format.html do
render layout: false
@@ -351,5 +360,104 @@ module Blazer
def blazer_run_id
params[:run_id].to_s.gsub(/[^a-z0-9\-]/i, "")
end
+
+ def run_cohort_analysis
+ unless @data_source.supports_cohort_analysis?
+ @cohort_error = "This data source does not support cohort analysis"
+ end
+
+ @show_cohort_rows = !params[:query_id] || @cohort_error
+
+ unless @show_cohort_rows
+ @statement = cohort_analysis_statement(@data_source, @statement)
+ end
+ end
+
+ def cohort_analysis_statement(data_source, statement)
+ @cohort_period = params["cohort_period"] || "week"
+ @cohort_period = "week" unless ["day", "week", "month"].include?(@cohort_period)
+
+ # for now
+ @conversion_period = @cohort_period
+ @cohort_days =
+ case @cohort_period
+ when "day"
+ 1
+ when "week"
+ 7
+ when "month"
+ 30
+ end
+
+ data_source.cohort_analysis_statement(statement, period: @cohort_period, days: @cohort_days)
+ end
+
+ def render_cohort_analysis
+ if @show_cohort_rows
+ @cohort_analysis = false
+
+ @row_limit = 1000
+
+ # check results
+ unless @cohort_error
+ # check names
+ expected_columns = ["user_id", "conversion_time"]
+ missing_columns = expected_columns - @result.columns
+ if missing_columns.any?
+ @cohort_error = "Expected user_id and conversion_time columns"
+ end
+
+ # check types (user_id can be any type)
+ unless @cohort_error
+ column_types = @result.columns.zip(@result.column_types).to_h
+
+ if !column_types["cohort_time"].in?(["time", nil])
+ @cohort_error = "cohort_time must be time column"
+ elsif !column_types["conversion_time"].in?(["time", nil])
+ @cohort_error = "conversion_time must be time column"
+ end
+ end
+ end
+ else
+ @today = Blazer.time_zone.today
+ @min_cohort_date, @max_cohort_date = @result.rows.map { |r| r[0] }.minmax
+ @buckets = {}
+ @rows.each do |r|
+ @buckets[[r[0], r[1]]] = r[2]
+ end
+
+ @cohort_dates = []
+ current_date = @max_cohort_date
+ while current_date && current_date >= @min_cohort_date
+ @cohort_dates << current_date
+ current_date =
+ case @cohort_period
+ when "day"
+ current_date - 1
+ when "week"
+ current_date - 7
+ else
+ current_date.prev_month
+ end
+ end
+
+ num_cols = @cohort_dates.size
+ @columns = ["Cohort", "Users"] + num_cols.times.map { |i| "#{@conversion_period.titleize} #{i + 1}" }
+ rows = []
+ date_format = @cohort_period == "month" ? "%b %Y" : "%b %-e, %Y"
+ @cohort_dates.each do |date|
+ row = [date.strftime(date_format), @buckets[[date, 0]] || 0]
+
+ num_cols.times do |i|
+ if @today >= date + (@cohort_days * i)
+ row << (@buckets[[date, i + 1]] || 0)
+ end
+ end
+
+ rows << row
+ end
+ @rows = rows
+ end
+ end
end
end
diff --git a/app/models/blazer/query.rb b/app/models/blazer/query.rb
index 0b392092..9f6f1690 100644
--- a/app/models/blazer/query.rb
+++ b/app/models/blazer/query.rb
@@ -35,7 +35,13 @@ module Blazer
end
def variables
- Blazer.extract_vars(statement)
+ variables = Blazer.extract_vars(statement)
+ variables += ["cohort_period"] if cohort_analysis?
+ variables
+ end
+
+ def cohort_analysis?
+ /\/\*\s*cohort analysis\s*\*\//i.match?(statement)
end
end
end
diff --git a/app/views/blazer/dashboards/show.html.erb b/app/views/blazer/dashboards/show.html.erb
index d35a5199..4878fb0b 100644
--- a/app/views/blazer/dashboards/show.html.erb
+++ b/app/views/blazer/dashboards/show.html.erb
@@ -39,7 +39,7 @@
</div>
</div>
<script>
- <%= blazer_js_var "data", {statement: @statements[i], query_id: query.id, data_source: query.data_source, only_chart: true} %>
+ <%= blazer_js_var "data", {statement: @statements[i], query_id: query.id, data_source: query.data_source, only_chart: true, cohort_period: params[:cohort_period]} %>
runQuery(data, function (data) {
$("#chart-<%= i %>").html(data)
diff --git a/app/views/blazer/queries/_caching.html.erb b/app/views/blazer/queries/_caching.html.erb
new file mode 100644
index 00000000..31e820e2
--- /dev/null
+++ b/app/views/blazer/queries/_caching.html.erb
@@ -0,0 +1,16 @@
+<% if @cached_at || @just_cached %>
+ <p class="text-muted" style="float: right;">
+ <% if @cached_at %>
+ Cached <%= time_ago_in_words(@cached_at, include_seconds: true) %> ago
+ <% elsif params[:query_id] %>
+ Cached just now
+ <% if @data_source.cache_mode == "slow" %>
+ (over <%= "%g" % @data_source.cache_slow_threshold %>s)
+ <% end %>
+ <% end %>
+
+ <% if @query && params[:query_id] %>
+ <%= link_to "Refresh", refresh_query_path(@query, variable_params(@query)), method: :post %>
+ <% end %>
+ </p>
+<% end %>
diff --git a/app/views/blazer/queries/_cohorts.html.erb b/app/views/blazer/queries/_cohorts.html.erb
new file mode 100644
index 00000000..ff434475
--- /dev/null
+++ b/app/views/blazer/queries/_cohorts.html.erb
@@ -0,0 +1,48 @@
+<% unless @only_chart %>
+ <%= render partial: "caching" %>
+ <p class="text-muted" style="margin-bottom: 10px;">
+ <%= pluralize(@rows.size, "cohort") %>
+ </p>
+<% end %>
+<% if @rows.any? %>
+ <div class="results-container">
+ <table class="table results-table">
+ <thead>
+ <tr>
+ <th style="min-width: 100px;">Cohort</th>
+ <% 12.times do |i| %>
+ <th style="width: 7.5%; text-align: right;"><%= @conversion_period.titleize %> <%= i + 1 %></th>
+ <% end %>
+ </tr>
+ </thead>
+ <tbody>
+ <% @rows.each do |row| %>
+ <tr>
+ <td>
+ <%= row[0] %>
+ <div style="font-size: 12px; color: #999;"><%= row[1] == 1 ? "1 user" : "#{number_with_delimiter(row[1])} users" %></div>
+ </td>
+ <% 12.times do |i| %>
+ <td style="text-align: right;">
+ <% num = row[i + 2] %>
+ <% if num %>
+ <% denom = row[1] %>
+ <% if denom > 0 %>
+ <%= (100.0 * num / denom).round %>%
+ <% else %>
+ -
+ <% end %>
+ <div style="font-size: 12px; color: #999;"><%= number_with_delimiter(num) %></div>
+ <% else %>
+ -
+ <% end %>
+ </td>
+ <% end %>
+ </tr>
+ <% end %>
+ </tbody>
+ </table>
+ </div>
+<% elsif @only_chart %>
+ <p class="text-muted">No cohorts</p>
+<% end %>
diff --git a/app/views/blazer/queries/docs.html.erb b/app/views/blazer/queries/docs.html.erb
index 503c46a2..ba5ea92b 100644
--- a/app/views/blazer/queries/docs.html.erb
+++ b/app/views/blazer/queries/docs.html.erb
@@ -129,3 +129,9 @@
</table>
<p>Use the column name <code>target</code> to draw a line for goals.</p>
+
+<% if @data_source.supports_cohort_analysis? %>
+ <h2>Cohort Analysis</h2>
+
+ <p>Create a query with the comment <code>/* cohort analysis */</code>. The result should have columns named <code>user_id</code> and <code>conversion_time</code> and optionally <code>cohort_time</code>.</p>
+<% end %>
diff --git a/app/views/blazer/queries/run.html.erb b/app/views/blazer/queries/run.html.erb
index 824f2f43..f6c571b2 100644
--- a/app/views/blazer/queries/run.html.erb
+++ b/app/views/blazer/queries/run.html.erb
@@ -6,25 +6,20 @@
<% else %>
<div class="alert alert-info">Can’t preview queries with variables...yet!</div>
<% end %>
+<% elsif @cohort_analysis %>
+ <% if @cohort_error %>
+ <div class="alert alert-info"><%= @cohort_error %></div>
+ <% else %>
+ <%= render partial: "cohorts" %>
+ <% end %>
<% else %>
<% unless @only_chart %>
- <% if @cached_at || @just_cached %>
- <p class="text-muted" style="float: right;">
- <% if @cached_at %>
- Cached <%= time_ago_in_words(@cached_at, include_seconds: true) %> ago
- <% elsif params[:query_id] %>
- Cached just now
- <% if @data_source.cache_mode == "slow" %>
- (over <%= "%g" % @data_source.cache_slow_threshold %>s)
- <% end %>
- <% end %>
-
- <% if @query && params[:query_id] %>
- <%= link_to "Refresh", refresh_query_path(@query, variable_params(@query)), method: :post %>
- <% end %>
- </p>
- <% end %>
+ <%= render partial: "caching" %>
<p class="text-muted" style="margin-bottom: 10px;">
+ <% if @row_limit && @rows.size > @row_limit %>
+ First
+ <% @rows = @rows.first(@row_limit) %>
+ <% end %>
<%= pluralize(@rows.size, "row") %>
<% @checks.select(&:state).each do |check| %>
@@ -43,6 +38,9 @@
<% if @forecast_error %>
<div class="alert alert-danger"><%= @forecast_error %></div>
<% end %>
+ <% if @cohort_error %>
+ <div class="alert alert-info"><%= @cohort_error %></div>
+ <% end %>
<% if @rows.any? %>
<% values = @rows.first %>
<% chart_id = SecureRandom.hex %>
@@ -147,7 +145,7 @@
<% elsif @columns == ["PLAN"] && @data_source.adapter == "druid" %>
<pre><code><%= @rows[0][0] %></code></pre>
<% else %>
- <table class="table results-table" style="margin-bottom: 0;">
+ <table class="table results-table">
<thead>
<tr>
<% @columns.each_with_index do |key, i| %>
diff --git a/app/views/blazer/queries/show.html.erb b/app/views/blazer/queries/show.html.erb
index afe42c31..c04a59bc 100644
--- a/app/views/blazer/queries/show.html.erb
+++ b/app/views/blazer/queries/show.html.erb
@@ -14,7 +14,7 @@
<%= link_to "Fork", new_query_path(variable_params(@query).merge(fork_query_id: @query.id, data_source: @query.data_source, name: @query.name)), class: "btn btn-info" %>
<% if !@error && @success %>
- <%= button_to "Download", run_queries_path(query_id: @query.id, format: "csv", forecast: params[:forecast]), params: {statement: @statement}, class: "btn btn-primary" %>
+ <%= button_to "Download", run_queries_path(query_id: @query.id, format: "csv", forecast: params[:forecast], cohort_period: params[:cohort_period]), params: {statement: @statement}, class: "btn btn-primary" %>
<% end %>
</div>
</div>
diff --git a/lib/blazer/adapters/base_adapter.rb b/lib/blazer/adapters/base_adapter.rb
index e9b78416..631c943e 100644
--- a/lib/blazer/adapters/base_adapter.rb
+++ b/lib/blazer/adapters/base_adapter.rb
@@ -43,6 +43,14 @@ module Blazer
true # optional
end
+ def supports_cohort_analysis?
+ false # optional
+ end
+
+ def cohort_analysis_statement(statement, period:, days:)
+ # optional
+ end
+
protected
def settings
diff --git a/lib/blazer/adapters/sql_adapter.rb b/lib/blazer/adapters/sql_adapter.rb
index ba26ef97..f5bd73a1 100644
--- a/lib/blazer/adapters/sql_adapter.rb
+++ b/lib/blazer/adapters/sql_adapter.rb
@@ -122,6 +122,47 @@ module Blazer
!%w[CREATE ALTER UPDATE INSERT DELETE].include?(statement.split.first.to_s.upcase)
end
+ def supports_cohort_analysis?
+ postgresql?
+ end
+
+ # TODO treat date columns as already in time zone
+ def cohort_analysis_statement(statement, period:, days:)
+ raise "Cohort analysis not supported" unless supports_cohort_analysis?
+
+ cohort_column = statement =~ /\bcohort_time\b/ ? "cohort_time" : "conversion_time"
+
+ # WITH not an optimization fence in Postgres 12+
+ statement = <<~SQL
+ WITH query AS (
+ #{statement}
+ ),
+ cohorts AS (
+ SELECT user_id, MIN(#{cohort_column}) AS cohort_time FROM query
+ WHERE user_id IS NOT NULL AND #{cohort_column} IS NOT NULL
+ GROUP BY 1
+ )
+ SELECT
+ date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date AS period,
+ 0 AS bucket,
+ COUNT(DISTINCT cohorts.user_id)
+ FROM cohorts GROUP BY 1
+ UNION ALL
+ SELECT
+ date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date AS period,
+ CEIL(EXTRACT(EPOCH FROM query.conversion_time - cohorts.cohort_time) / ?)::int AS bucket,
+ COUNT(DISTINCT query.user_id)
+ FROM cohorts INNER JOIN query ON query.user_id = cohorts.user_id
+ WHERE query.conversion_time IS NOT NULL
+ AND query.conversion_time >= cohorts.cohort_time
+ #{cohort_column == "conversion_time" ? "AND query.conversion_time != cohorts.cohort_time" : ""}
+ GROUP BY 1, 2
+ SQL
+ tzname = Blazer.time_zone.tzinfo.name
+ params = [statement, period, tzname, period, tzname, days.to_i * 86400]
+ connection_model.send(:sanitize_sql_array, params)
+ end
+
protected
def select_all(statement, params = [])
diff --git a/lib/blazer/data_source.rb b/lib/blazer/data_source.rb
index 8842d341..8ad0fe2d 100644
--- a/lib/blazer/data_source.rb
+++ b/lib/blazer/data_source.rb
@@ -6,7 +6,7 @@ module Blazer
attr_reader :id, :settings
- def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel
+ def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel, :supports_cohort_analysis?, :cohort_analysis_statement
def initialize(id, settings)
@id = id