Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
APIv2: TimeSeries using QueryBuilder, release `experimental_session_count` (#4305)
Karl-Aksel Puulmanna181f3eab34a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 747f7cfe35..7b6dc22a53 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -14,6 +14,7 @@ All notable changes to this project will be documented in this file.### Changed+- Realtime and hourly graphs now show visits lasting their whole duration instead when specific events occur- Increase hourly request limit for API keys in CE from 600 to 1000000 (practically removing the limit) plausible/analytics#4200- Make TCP connections try IPv6 first with IPv4 fallback in CE plausible/analytics#4245- `is` and `is not` filters in dashboard no longer support wildcards. Use contains/does not contain filter instead.diff --git a/assets/js/dashboard/api.js b/assets/js/dashboard/api.jsindex 6aaae36339..a9ff924e44 100644--- a/assets/js/dashboard/api.js+++ b/assets/js/dashboard/api.js@@ -38,7 +38,6 @@ export function serializeQuery(query, extraQuery = []) {if (query.from) { queryObj.from = formatISO(query.from) }if (query.to) { queryObj.to = formatISO(query.to) }if (query.filters) { queryObj.filters = serializeApiFilters(query.filters) }- if (query.experimental_session_count) { queryObj.experimental_session_count = query.experimental_session_count }if (query.with_imported) { queryObj.with_imported = query.with_imported }if (SHARED_LINK_AUTH) { queryObj.auth = SHARED_LINK_AUTH }diff --git a/assets/js/dashboard/query.js b/assets/js/dashboard/query.jsindex cf702243af..2e18fd5de9 100644--- a/assets/js/dashboard/query.js+++ b/assets/js/dashboard/query.js@@ -42,7 +42,6 @@ export function parseQuery(querystring, site) {to: q.get('to') ? dayjs.utc(q.get('to')) : undefined,match_day_of_week: matchDayOfWeek == 'true',with_imported: q.get('with_imported') ? q.get('with_imported') === 'true' : true,- experimental_session_count: q.get('experimental_session_count'),filters: parseJsonUrl(q.get('filters'), []),labels: parseJsonUrl(q.get('labels'), {})}diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 8335308cce..cb104ffc76 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -139,84 +139,4 @@ defmodule Plausible.Stats.Base do"^#{escaped}$"end-- defp total_visitors(site, query) do- base_event_query(site, query)- |> select([e],- total_visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)- )- end-- # `total_visitors_subquery` returns a subquery which selects `total_visitors` -- # the number used as the denominator in the calculation of `conversion_rate` and- # `percentage` metrics.-- # Usually, when calculating the totals, a new query is passed into this function,- # where certain filters (e.g. goal, props) are removed. That might make the query- # able to include imported data. However, we always want to include imported data- # only if it's included in the base query - otherwise the total will be based on- # a different data set, making the metric inaccurate. This is why we're using an- # explicit `include_imported` argument here.- def total_visitors_subquery(site, query, include_imported)-- def total_visitors_subquery(site, query, true = _include_imported) do- wrap_alias([], %{- total_visitors:- subquery(total_visitors(site, query)) +- subquery(Plausible.Stats.Imported.total_imported_visitors(site, query))- })- end-- def total_visitors_subquery(site, query, false = _include_imported) do- wrap_alias([], %{- total_visitors: subquery(total_visitors(site, query))- })- end-- def add_percentage_metric(q, site, query, metrics) do- if :percentage in metrics do- total_query = Query.set_dimensions(query, [])-- q- |> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported))- |> select_merge_as([], %{- percentage:- fragment(- "if(? > 0, round(? / ? * 100, 1), null)",- selected_as(:total_visitors),- selected_as(:visitors),- selected_as(:total_visitors)- )- })- else- q- end- end-- # Adds conversion_rate metric to query, calculated as- # X / Y where Y is the same breakdown value without goal or props- # filters.- def maybe_add_conversion_rate(q, site, query, metrics) do- if :conversion_rate in metrics do- total_query =- query- |> Query.remove_filters(["event:goal", "event:props"])- |> Query.set_dimensions([])-- # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL- subquery(q)- |> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported))- |> select_merge_as([e], %{- conversion_rate:- fragment(- "if(? > 0, round(? / ? * 100, 1), 0)",- selected_as(:total_visitors),- e.visitors,- selected_as(:total_visitors)- )- })- else- q- end- endenddiff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.exindex 4993c6dd33..157be97c1f 100644--- a/lib/plausible/stats/comparisons.ex+++ b/lib/plausible/stats/comparisons.ex@@ -163,21 +163,21 @@ defmodule Plausible.Stats.Comparisons doenddefp maybe_include_imported(query, source_query) do- requested? = source_query.imported_data_requested+ requested? = source_query.include.importscase Query.ensure_include_imported(query, requested?) do:ok ->struct!(query,- imported_data_requested: true,include_imported: true,- skip_imported_reason: nil+ skip_imported_reason: nil,+ include: Map.put(query.include, :imports, true)){:error, reason} ->struct!(query,- imported_data_requested: requested?,include_imported: false,- skip_imported_reason: reason+ skip_imported_reason: reason,+ include: Map.put(query.include, :imports, requested?))endenddiff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.exindex b6760965ed..60c1575fc8 100644--- a/lib/plausible/stats/filters/filters.ex+++ b/lib/plausible/stats/filters/filters.ex@@ -83,8 +83,8 @@ defmodule Plausible.Stats.Filters dodef parse(_), do: []- def without_prefix(property) do- property+ def without_prefix(dimension) do+ dimension|> String.split(":")|> List.last()|> String.to_existing_atom()diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex 3f802dcb64..0ed934fd68 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -6,6 +6,11 @@ defmodule Plausible.Stats.Filters.QueryParser doalias Plausible.Stats.Queryalias Plausible.Stats.Metrics+ @default_include %{+ imports: false,+ time_labels: false+ }+def parse(site, params, now \\ nil) when is_map(params) dowith {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),{:ok, filters} <- parse_filters(Map.get(params, "filters", [])),@@ -22,13 +27,14 @@ defmodule Plausible.Stats.Filters.QueryParser dodimensions: dimensions,order_by: order_by,timezone: site.timezone,- imported_data_requested: Map.get(include, :imports, false),- preloaded_goals: preloaded_goals+ preloaded_goals: preloaded_goals,+ include: include},:ok <- validate_order_by(query),:ok <- validate_goal_filters(query),:ok <- validate_custom_props_access(site, query),- :ok <- validate_metrics(query) do+ :ok <- validate_metrics(query),+ :ok <- validate_include(query) do{:ok, query}endend@@ -219,6 +225,7 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_time("time"), do: {:ok, "time"}defp parse_time("time:hour"), do: {:ok, "time:hour"}defp parse_time("time:day"), do: {:ok, "time:day"}+ defp parse_time("time:week"), do: {:ok, "time:week"}defp parse_time("time:month"), do: {:ok, "time:month"}defp parse_time(_), do: :error@@ -226,9 +233,24 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_order_direction([_, "desc"]), do: {:ok, :desc}defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{inspect(entry)}'"}- defp parse_include(%{"imports" => value}) when is_boolean(value), do: {:ok, %{imports: value}}- defp parse_include(%{}), do: {:ok, %{}}- defp parse_include(include), do: {:error, "Invalid include passed '#{inspect(include)}'"}+ defp parse_include(include) when is_map(include) do+ with {:ok, parsed_include_list} <- parse_list(include, &parse_include_value/1) do+ include = Map.merge(@default_include, Enum.into(parsed_include_list, %{}))++ {:ok, include}+ end+ end++ defp parse_include(entry), do: {:error, "Invalid include passed '#{inspect(entry)}'"}++ defp parse_include_value({"imports", value}) when is_boolean(value),+ do: {:ok, {:imports, value}}++ defp parse_include_value({"time_labels", value}) when is_boolean(value),+ do: {:ok, {:time_labels, value}}++ defp parse_include_value({key, value}),+ do: {:error, "Invalid include entry '#{inspect(%{key => value})}'"}defp parse_filter_key_string(filter_key, error_message \\ "") docase filter_key do@@ -386,7 +408,7 @@ defmodule Plausible.Stats.Filters.QueryParser doendend- def event_dimensions_not_allowing_session_metrics?(dimensions) do+ defp event_dimensions_not_allowing_session_metrics?(dimensions) doEnum.any?(dimensions, fn"event:page" -> false"event:" <> _ -> true@@ -394,6 +416,16 @@ defmodule Plausible.Stats.Filters.QueryParser doend)end+ defp validate_include(query) do+ time_dimension? = Enum.any?(query.dimensions, &String.starts_with?(&1, "time"))++ if query.include.time_labels and not time_dimension? do+ {:error, "Invalid include.time_labels: requires a time dimension"}+ else+ :ok+ end+ end+defp parse_list(list, parser_function) doEnum.reduce_while(list, {:ok, []}, fn value, {:ok, results} ->case parser_function.(value) dodiff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.exindex 72a888d186..11909b234b 100644--- a/lib/plausible/stats/imported/base.ex+++ b/lib/plausible/stats/imported/base.ex@@ -29,11 +29,19 @@ defmodule Plausible.Stats.Imported.Base do"event:page" => "imported_pages","event:name" => "imported_custom_events",- # NOTE: these properties can be only filtered by+ # NOTE: these dimensions can be only filtered by"visit:screen" => "imported_devices",- "event:hostname" => "imported_pages"+ "event:hostname" => "imported_pages",++ # NOTE: These dimensions are only used in group by+ "time:month" => "imported_visitors",+ "time:week" => "imported_visitors",+ "time:day" => "imported_visitors",+ "time:hour" => "imported_visitors"}+ @queriable_time_dimensions ["time:month", "time:week", "time:day", "time:hour"]+@imported_custom_props Imported.imported_custom_props()@db_field_mappings %{@@ -121,9 +129,10 @@ defmodule Plausible.Stats.Imported.Base dodo_decide_custom_prop_table(query, dimension)end+ @queriable_custom_prop_dimensions ["event:goal", "event:name"] ++ @queriable_time_dimensionsdefp do_decide_custom_prop_table(%{dimensions: dimensions} = query) doif dimensions == [] or- (length(dimensions) == 1 and hd(dimensions) in ["event:goal", "event:name"]) do+ (length(dimensions) == 1 and hd(dimensions) in @queriable_custom_prop_dimensions) docustom_prop_filters =query.filters|> Enum.map(&Enum.at(&1, 1))@@ -169,14 +178,6 @@ defmodule Plausible.Stats.Imported.Base do["imported_pages", "imported_custom_events"]end- defp do_decide_tables(%Query{filters: [], dimensions: [dimension]}) do- if Map.has_key?(@property_to_table_mappings, dimension) do- [@property_to_table_mappings[dimension]]- else- []- end- end-defp do_decide_tables(%Query{filters: filters, dimensions: ["event:goal"]}) dofilter_props = Enum.map(filters, &Enum.at(&1, 1))@@ -197,13 +198,15 @@ defmodule Plausible.Stats.Imported.Base dofilters|> Enum.map(fn [_, filter_key | _] -> filter_key end)|> Enum.concat(dimensions)- |> Enum.map(fn- "visit:screen" -> "visit:device"- dimension -> dimension+ |> Enum.reject(&(&1 in @queriable_time_dimensions))+ |> Enum.flat_map(fn+ "visit:screen" -> ["visit:device"]+ dimension -> [dimension]end)|> Enum.map(&@property_to_table_mappings[&1])case Enum.uniq(table_candidates) do+ [] -> ["imported_visitors"][nil] -> [][candidate] -> [candidate]_ -> []diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.exindex 4fcdd04a66..a1e817e484 100644--- a/lib/plausible/stats/imported/imported.ex+++ b/lib/plausible/stats/imported/imported.ex@@ -16,9 +16,6 @@ defmodule Plausible.Stats.Imported do@property_to_table_mappings Imported.Base.property_to_table_mappings()- @imported_dimensions Map.keys(@property_to_table_mappings) ++- Plausible.Imported.imported_custom_props()-@goals_with_url Plausible.Imported.goals_with_url()def goals_with_url(), do: @goals_with_url@@ -225,47 +222,6 @@ defmodule Plausible.Stats.Imported do{table, db_field}end- def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),- do: native_q-- def merge_imported_timeseries(- native_q,- site,- query,- metrics- ) do- imported_q =- site- |> Imported.Base.query_imported(query)- |> select_imported_metrics(metrics)- |> apply_interval(query, site)-- from(s in Ecto.Query.subquery(native_q),- full_join: i in subquery(imported_q),- on: s.date == i.date,- select: %{date: fragment("greatest(?, ?)", s.date, i.date)}- )- |> select_joined_metrics(metrics)- end-- defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}, _site) do- imported_q- |> group_by([i], fragment("toStartOfMonth(?)", i.date))- |> select_merge([i], %{date: fragment("toStartOfMonth(?)", i.date)})- end-- defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "week"} = query, _site) do- imported_q- |> group_by([i], weekstart_not_before(i.date, ^query.date_range.first))- |> select_merge([i], %{date: weekstart_not_before(i.date, ^query.date_range.first)})- end-- defp apply_interval(imported_q, _query, _site) do- imported_q- |> group_by([i], i.date)- |> select_merge([i], %{date: i.date})- end-def merge_imported(q, _, %Query{include_imported: false}, _), do: qdef merge_imported(q, site, %Query{dimensions: []} = query, metrics) do@@ -320,8 +276,8 @@ defmodule Plausible.Stats.Imported doend)end- def merge_imported(q, site, %Query{dimensions: dimensions} = query, metrics) do- if merge_imported_dimensions?(dimensions) do+ def merge_imported(q, site, query, metrics) do+ if schema_supports_query?(query) doimported_q =site|> Imported.Base.query_imported(query)@@ -341,13 +297,6 @@ defmodule Plausible.Stats.Imported doendend- def merge_imported(q, _, _, _), do: q-- defp merge_imported_dimensions?(dimensions) do- dimensions in [["visit:browser", "visit:browser_version"], ["visit:os", "visit:os_version"]] or- (length(dimensions) == 1 and hd(dimensions) in @imported_dimensions)- end-def total_imported_visitors(site, query) dosite|> Imported.Base.query_imported(query)@@ -552,11 +501,11 @@ defmodule Plausible.Stats.Imported doEnum.reduce(query.dimensions, q, fn dimension, q ->dim = Plausible.Stats.Filters.without_prefix(dimension)- group_imported_by(q, dim, shortname(query, dimension))+ group_imported_by(q, dim, shortname(query, dimension), query)end)end- defp group_imported_by(q, dim, key) when dim in [:source, :referrer] do+ defp group_imported_by(q, dim, key, _query) when dim in [:source, :referrer] doq|> group_by([i], field(i, ^dim))|> select_merge_as([i], %{@@ -564,7 +513,7 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, dim, key)+ defp group_imported_by(q, dim, key, _query)when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] doq|> group_by([i], field(i, ^dim))@@ -572,34 +521,34 @@ defmodule Plausible.Stats.Imported do|> select_merge_as([i], %{key => field(i, ^dim)})end- defp group_imported_by(q, :page, key) do+ defp group_imported_by(q, :page, key, _query) doq|> group_by([i], i.page)|> select_merge_as([i], %{key => i.page, time_on_page: sum(i.time_on_page)})end- defp group_imported_by(q, :country, key) do+ defp group_imported_by(q, :country, key, _query) doq|> group_by([i], i.country)|> where([i], i.country != "ZZ")|> select_merge_as([i], %{key => i.country})end- defp group_imported_by(q, :region, key) do+ defp group_imported_by(q, :region, key, _query) doq|> group_by([i], i.region)|> where([i], i.region != "")|> select_merge_as([i], %{key => i.region})end- defp group_imported_by(q, :city, key) do+ defp group_imported_by(q, :city, key, _query) doq|> group_by([i], i.city)|> where([i], i.city != 0 and not is_nil(i.city))|> select_merge_as([i], %{key => i.city})end- defp group_imported_by(q, dim, key) when dim in [:device, :browser] do+ defp group_imported_by(q, dim, key, _query) when dim in [:device, :browser] doq|> group_by([i], field(i, ^dim))|> select_merge_as([i], %{@@ -607,7 +556,7 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, :browser_version, key) do+ defp group_imported_by(q, :browser_version, key, _query) doq|> group_by([i], [i.browser_version])|> select_merge_as([i], %{@@ -615,7 +564,7 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, :os, key) do+ defp group_imported_by(q, :os, key, _query) doq|> group_by([i], i.operating_system)|> select_merge_as([i], %{@@ -623,7 +572,7 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, :os_version, key) do+ defp group_imported_by(q, :os_version, key, _query) doq|> group_by([i], [i.operating_system_version])|> select_merge_as([i], %{@@ -637,19 +586,19 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, dim, key) when dim in [:entry_page, :exit_page] do+ defp group_imported_by(q, dim, key, _query) when dim in [:entry_page, :exit_page] doq|> group_by([i], field(i, ^dim))|> select_merge_as([i], %{key => field(i, ^dim)})end- defp group_imported_by(q, :name, key) do+ defp group_imported_by(q, :name, key, _query) doq|> group_by([i], i.name)|> select_merge_as([i], %{key => i.name})end- defp group_imported_by(q, :url, key) do+ defp group_imported_by(q, :url, key, _query) doq|> group_by([i], i.link_url)|> select_merge_as([i], %{@@ -657,7 +606,7 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, :path, key) do+ defp group_imported_by(q, :path, key, _query) doq|> group_by([i], i.path)|> select_merge_as([i], %{@@ -665,6 +614,32 @@ defmodule Plausible.Stats.Imported do})end+ defp group_imported_by(q, :month, key, _query) do+ q+ |> group_by([i], fragment("toStartOfMonth(?)", i.date))+ |> select_merge_as([i], %{key => fragment("toStartOfMonth(?)", i.date)})+ end++ defp group_imported_by(q, :hour, key, _query) do+ q+ |> group_by([i], i.date)+ |> select_merge_as([i], %{key => i.date})+ end++ defp group_imported_by(q, :week, key, query) do+ q+ |> group_by([i], weekstart_not_before(i.date, ^query.date_range.first))+ |> select_merge_as([i], %{+ key => weekstart_not_before(i.date, ^query.date_range.first)+ })+ end++ defp group_imported_by(q, :day, key, _query) do+ q+ |> group_by([i], i.date)+ |> select_merge_as([i], %{key => i.date})+ end+defp select_joined_dimensions(q, query) doEnum.reduce(query.dimensions, q, fn dimension, q ->select_joined_dimension(q, dimension, shortname(query, dimension))@@ -677,6 +652,12 @@ defmodule Plausible.Stats.Imported do})end+ defp select_joined_dimension(q, "time:" <> _, key) do+ select_merge_as(q, [s, i], %{+ key => fragment("greatest(?, ?)", field(i, ^key), field(s, ^key))+ })+ end+defp select_joined_dimension(q, _dimension, key) doselect_merge_as(q, [s, i], %{key => fragment("if(empty(?), ?, ?)", field(s, ^key), field(i, ^key), field(s, ^key))diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 75fb8c503c..9d2eb0d667 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -7,11 +7,9 @@ defmodule Plausible.Stats.Query dodimensions: [],filters: [],sample_threshold: 20_000_000,- imported_data_requested: false,include_imported: false,skip_imported_reason: nil,now: nil,- experimental_session_count?: false,experimental_reduced_joins?: false,latest_import_end_date: nil,metrics: [],@@ -36,8 +34,7 @@ defmodule Plausible.Stats.Query doquery =__MODULE__- |> struct!(now: now)- |> put_experimental_session_count(site, params)+ |> struct!(now: now, timezone: site.timezone)|> put_experimental_reduced_joins(site, params)|> put_period(site, params)|> put_dimensions(params)@@ -57,7 +54,6 @@ defmodule Plausible.Stats.Query doquery =struct!(__MODULE__, Map.to_list(query_data))|> put_imported_opts(site, %{})- |> put_experimental_session_count(site, params)|> put_experimental_reduced_joins(site, params)|> struct!(v2: true)@@ -65,18 +61,6 @@ defmodule Plausible.Stats.Query doendend- defp put_experimental_session_count(query, site, params) do- if Map.has_key?(params, "experimental_session_count") do- struct!(query,- experimental_session_count?: Map.get(params, "experimental_session_count") == "true"- )- else- struct!(query,- experimental_session_count?: FunWithFlags.enabled?(:experimental_session_count, for: site)- )- end- end-defp put_experimental_reduced_joins(query, site, params) doif Map.has_key?(params, "experimental_reduced_joins") dostruct!(query,@@ -231,9 +215,13 @@ defmodule Plausible.Stats.Query doenddef set(query, keywords) do- query- |> struct!(keywords)- |> refresh_imported_opts()+ new_query = struct!(query, keywords)++ if Keyword.has_key?(keywords, :include_imported) do+ new_query+ else+ refresh_imported_opts(new_query)+ endend@spec set_dimensions(t(), list(String.t())) :: t()@@ -314,7 +302,7 @@ defmodule Plausible.Stats.Query doenddefp put_imported_opts(query, site, params) do- requested? = params["with_imported"] == "true" || query.imported_data_requested+ requested? = params["with_imported"] == "true" || query.include.importslatest_import_end_date =if site do@@ -328,15 +316,15 @@ defmodule Plausible.Stats.Query docase ensure_include_imported(query, requested?) do:ok ->struct!(query,- imported_data_requested: true,- include_imported: true+ include_imported: true,+ include: Map.put(query.include, :imports, true)){:error, reason} ->struct!(query,- imported_data_requested: requested?,include_imported: false,- skip_imported_reason: reason+ skip_imported_reason: reason,+ include: Map.put(query.include, :imports, requested?))endenddiff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.exindex 25146d950d..5f3bf706f8 100644--- a/lib/plausible/stats/query_optimizer.ex+++ b/lib/plausible/stats/query_optimizer.ex@@ -33,7 +33,7 @@ defmodule Plausible.Stats.QueryOptimizer do|> TableDecider.partition_metrics(query){- Query.set_metrics(query, event_metrics),+ Query.set(query, metrics: event_metrics, include_imported: query.include_imported),split_sessions_query(query, sessions_metrics)}end@@ -80,6 +80,7 @@ defmodule Plausible.Stats.QueryOptimizer docond doTimex.diff(last, first, :hours) <= 48 -> "time:hour"Timex.diff(last, first, :days) <= 40 -> "time:day"+ Timex.diff(last, first, :weeks) <= 52 -> "time:week"true -> "time:month"endend@@ -160,6 +161,11 @@ defmodule Plausible.Stats.QueryOptimizer doquery.filtersend- Query.set(query, filters: filters, metrics: session_metrics, dimensions: dimensions)+ Query.set(query,+ filters: filters,+ metrics: session_metrics,+ dimensions: dimensions,+ include_imported: query.include_imported+ )endenddiff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.exindex b9f3744448..c59d22ddcf 100644--- a/lib/plausible/stats/query_result.ex+++ b/lib/plausible/stats/query_result.ex@@ -3,7 +3,6 @@ defmodule Plausible.Stats.QueryResult doalias Plausible.Stats.Utilalias Plausible.Stats.Filters- alias Plausible.Stats.Query@derive Jason.Encoderdefstruct results: [],@@ -34,16 +33,6 @@ defmodule Plausible.Stats.QueryResult do)end- defp meta(%Query{skip_imported_reason: :unsupported_query}) do- %{- warning:- "Imported stats are not included in the results because query parameters are not supported. " <>- "For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats"- }- end-- defp meta(_), do: %{}-defp dimension_label("event:goal", entry, query) do{events, paths} = Filters.Utils.split_goals(query.preloaded_goals)@@ -56,6 +45,12 @@ defmodule Plausible.Stats.QueryResult doendend+ defp dimension_label("time:" <> _ = time_dimension, entry, query) do+ datetime = Map.get(entry, Util.shortname(query, time_dimension))++ Plausible.Stats.Time.format_datetime(datetime)+ end+defp dimension_label(dimension, entry, query) doMap.get(entry, Util.shortname(query, dimension))end@@ -65,4 +60,21 @@ defmodule Plausible.Stats.QueryResult doenddefp serializable_filter(filter), do: filter++ @imports_unsupported_query_warning "Imported stats are not included in the results because query parameters are not supported. " <>+ "For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats"++ defp meta(query) do+ %{+ warning:+ case query.skip_imported_reason do+ :unsupported_query -> @imports_unsupported_query_warning+ _ -> nil+ end,+ time_labels:+ if(query.include.time_labels, do: Plausible.Stats.Time.time_labels(query), else: nil)+ }+ |> Enum.reject(fn {_, value} -> is_nil(value) end)+ |> Enum.into(%{})+ endenddiff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exindex daea6791f4..b3b8a8dab1 100644--- a/lib/plausible/stats/sql/expression.ex+++ b/lib/plausible/stats/sql/expression.ex@@ -26,34 +26,98 @@ defmodule Plausible.Stats.SQL.Expression doendend- def dimension(key, "time:hour", query) do+ defmacrop regular_time_slots(query, period_in_seconds) do+ quote do+ fragment(+ "arrayJoin(timeSlots(toTimeZone(?, ?), toUInt32(timeDiff(?, ?)), toUInt32(?)))",+ s.start,+ ^unquote(query).timezone,+ s.start,+ s.timestamp,+ ^unquote(period_in_seconds)+ )+ end+ end++ def dimension(key, "time:month", _table, query) dowrap_alias([t], %{- key => fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone)+ key => fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone)+ })+ end++ def dimension(key, "time:week", _table, query) do+ wrap_alias([t], %{+ key =>+ weekstart_not_before(+ to_timezone(t.timestamp, ^query.timezone),+ ^query.date_range.first+ )})end- def dimension(key, "time:day", query) do+ def dimension(key, "time:day", _table, query) dowrap_alias([t], %{key => fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone)})end- def dimension(key, "time:month", query) do+ def dimension(key, "time:hour", :sessions, query) do+ wrap_alias([s], %{+ key => regular_time_slots(query, 3600)+ })+ end++ def dimension(key, "time:hour", _table, query) dowrap_alias([t], %{- key => fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone)+ key => fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone)+ })+ end++ # :NOTE: This is not exposed in Query APIv2+ def dimension(key, "time:minute", :sessions, %Query{+ period: "30m"+ }) do+ wrap_alias([s], %{+ key =>+ fragment(+ "arrayJoin(range(dateDiff('minute', now(), ?), dateDiff('minute', now(), ?) + 1))",+ s.start,+ s.timestamp+ )+ })+ end++ # :NOTE: This is not exposed in Query APIv2+ def dimension(key, "time:minute", _table, %Query{period: "30m"}) do+ wrap_alias([t], %{+ key => fragment("dateDiff('minute', now(), ?)", t.timestamp)+ })+ end++ # :NOTE: This is not exposed in Query APIv2+ def dimension(key, "time:minute", :sessions, query) do+ wrap_alias([s], %{+ key => regular_time_slots(query, 60)+ })+ end++ # :NOTE: This is not exposed in Query APIv2+ def dimension(key, "time:minute", _table, query) do+ wrap_alias([t], %{+ key => fragment("toStartOfMinute(toTimeZone(?, ?))", t.timestamp, ^query.timezone)})end- def dimension(key, "event:name", _query),+ def dimension(key, "event:name", _table, _query),do: wrap_alias([t], %{key => t.name})- def dimension(key, "event:page", _query),+ def dimension(key, "event:page", _table, _query),do: wrap_alias([t], %{key => t.pathname})- def dimension(key, "event:hostname", _query),+ def dimension(key, "event:hostname", _table, _query),do: wrap_alias([t], %{key => t.hostname})- def dimension(key, "event:props:" <> property_name, _query) do+ def dimension(key, "event:props:" <> property_name, _table, _query) dowrap_alias([t], %{key =>fragment(@@ -64,55 +128,55 @@ defmodule Plausible.Stats.SQL.Expression do})end- def dimension(key, "visit:entry_page", _query),+ def dimension(key, "visit:entry_page", _table, _query),do: wrap_alias([t], %{key => t.entry_page})- def dimension(key, "visit:exit_page", _query),+ def dimension(key, "visit:exit_page", _table, _query),do: wrap_alias([t], %{key => t.exit_page})- def dimension(key, "visit:utm_medium", _query),+ def dimension(key, "visit:utm_medium", _table, _query),do: field_or_blank_value(key, t.utm_medium, @not_set)- def dimension(key, "visit:utm_source", _query),+ def dimension(key, "visit:utm_source", _table, _query),do: field_or_blank_value(key, t.utm_source, @not_set)- def dimension(key, "visit:utm_campaign", _query),+ def dimension(key, "visit:utm_campaign", _table, _query),do: field_or_blank_value(key, t.utm_campaign, @not_set)- def dimension(key, "visit:utm_content", _query),+ def dimension(key, "visit:utm_content", _table, _query),do: field_or_blank_value(key, t.utm_content, @not_set)- def dimension(key, "visit:utm_term", _query),+ def dimension(key, "visit:utm_term", _table, _query),do: field_or_blank_value(key, t.utm_term, @not_set)- def dimension(key, "visit:source", _query),+ def dimension(key, "visit:source", _table, _query),do: field_or_blank_value(key, t.source, @no_ref)- def dimension(key, "visit:referrer", _query),+ def dimension(key, "visit:referrer", _table, _query),do: field_or_blank_value(key, t.referrer, @no_ref)- def dimension(key, "visit:device", _query),+ def dimension(key, "visit:device", _table, _query),do: field_or_blank_value(key, t.device, @not_set)- def dimension(key, "visit:os", _query),+ def dimension(key, "visit:os", _table, _query),do: field_or_blank_value(key, t.os, @not_set)- def dimension(key, "visit:os_version", _query),+ def dimension(key, "visit:os_version", _table, _query),do: field_or_blank_value(key, t.os_version, @not_set)- def dimension(key, "visit:browser", _query),+ def dimension(key, "visit:browser", _table, _query),do: field_or_blank_value(key, t.browser, @not_set)- def dimension(key, "visit:browser_version", _query),+ def dimension(key, "visit:browser_version", _table, _query),do: field_or_blank_value(key, t.browser_version, @not_set)- def dimension(key, "visit:country", _query),+ def dimension(key, "visit:country", _table, _query),do: wrap_alias([t], %{key => t.country})- def dimension(key, "visit:region", _query),+ def dimension(key, "visit:region", _table, _query),do: wrap_alias([t], %{key => t.region})- def dimension(key, "visit:city", _query),+ def dimension(key, "visit:city", _table, _query),do: wrap_alias([t], %{key => t.city})def event_metric(:pageviews) dodiff --git a/lib/plausible/stats/sql/fragments.ex b/lib/plausible/stats/sql/fragments.exindex 526682f9ef..5e0b0f6d18 100644--- a/lib/plausible/stats/sql/fragments.ex+++ b/lib/plausible/stats/sql/fragments.ex@@ -97,19 +97,6 @@ defmodule Plausible.Stats.SQL.Fragments doendend- @doc """- Same as Plausible.Stats.SQL.Fragments.weekstart_not_before/2 but converts dates to- the specified timezone.- """- defmacro weekstart_not_before(date, not_before, timezone) do- quote do- weekstart_not_before(- to_timezone(unquote(date), unquote(timezone)),- to_timezone(unquote(not_before), unquote(timezone))- )- end- end-@doc """Returns whether a key (usually property) exists under `meta.key` array or similar.@@ -192,6 +179,21 @@ defmodule Plausible.Stats.SQL.Fragments doendend+ @doc """+ Macro that helps join two Ecto queries by selecting fields from either one+ """+ defmacro select_join_fields(q, query, list, table_name) do+ quote do+ Enum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q ->+ key = shortname(unquote(query), metric_or_dimension)++ select_merge_as(q, [e, s], %{+ key => field(unquote(table_name), ^key)+ })+ end)+ end+ end+defp update_literal_map_values({:%{}, ctx, keyword_list}, mapper_fn) do{:%{},diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex 8cf92725bc..ef61a59758 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -41,11 +41,9 @@ defmodule Plausible.Stats.SQL.QueryBuilder doq|> join_sessions_if_needed(site, events_query)- |> build_group_by(events_query)+ |> build_group_by(:events, events_query)|> merge_imported(site, events_query, events_query.metrics)- |> maybe_add_global_conversion_rate(site, events_query)- |> maybe_add_group_conversion_rate(site, events_query)- |> Base.add_percentage_metric(site, events_query, events_query.metrics)+ |> SQL.SpecialMetrics.add(site, events_query)enddefp join_sessions_if_needed(q, site, query) do@@ -84,11 +82,9 @@ defmodule Plausible.Stats.SQL.QueryBuilder doq|> join_events_if_needed(site, sessions_query)- |> build_group_by(sessions_query)+ |> build_group_by(:sessions, sessions_query)|> merge_imported(site, sessions_query, sessions_query.metrics)- |> maybe_add_global_conversion_rate(site, sessions_query)- |> maybe_add_group_conversion_rate(site, sessions_query)- |> Base.add_percentage_metric(site, sessions_query, sessions_query.metrics)+ |> SQL.SpecialMetrics.add(site, sessions_query)enddef join_events_if_needed(q, site, query) do@@ -115,11 +111,11 @@ defmodule Plausible.Stats.SQL.QueryBuilder doendend- defp build_group_by(q, query) do- Enum.reduce(query.dimensions, q, &dimension_group_by(&2, query, &1))+ defp build_group_by(q, table, query) do+ Enum.reduce(query.dimensions, q, &dimension_group_by(&2, table, query, &1))end- defp dimension_group_by(q, query, "event:goal" = dimension) do+ defp dimension_group_by(q, _table, query, "event:goal" = dimension) do{events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)from(e in q,@@ -132,11 +128,11 @@ defmodule Plausible.Stats.SQL.QueryBuilder do)end- defp dimension_group_by(q, query, dimension) do+ defp dimension_group_by(q, table, query, dimension) dokey = shortname(query, dimension)q- |> select_merge_as([], Expression.dimension(key, dimension, query))+ |> select_merge_as([], Expression.dimension(key, dimension, table, query))|> group_by([], selected_as(^key))end@@ -155,87 +151,6 @@ defmodule Plausible.Stats.SQL.QueryBuilder do)end- defmacrop select_join_fields(q, query, list, table_name) do- quote do- Enum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q ->- key = shortname(unquote(query), metric_or_dimension)-- select_merge_as(q, [e, s], %{- key => field(unquote(table_name), ^key)- })- end)- end- end-- # Adds conversion_rate metric to query, calculated as- # X / Y where Y is the same breakdown value without goal or props- # filters.- def maybe_add_global_conversion_rate(q, site, query) do- if :conversion_rate in query.metrics do- total_query =- query- |> Query.remove_filters(["event:goal", "event:props"])- |> Query.set_dimensions([])-- q- |> select_merge_as(- [],- Base.total_visitors_subquery(site, total_query, query.include_imported)- )- |> select_merge_as([e], %{- conversion_rate:- fragment(- "if(? > 0, round(? / ? * 100, 1), 0)",- selected_as(:total_visitors),- selected_as(:visitors),- selected_as(:total_visitors)- )- })- else- q- end- end-- # This function injects a group_conversion_rate metric into- # a dimensional query. It is calculated as X / Y, where:- #- # * X is the number of conversions for a set of dimensions- # result (conversion = number of visitors who- # completed the filtered goal with the filtered- # custom properties).- #- # * Y is the number of all visitors for this set of dimensions- # result without the `event:goal` and `event:props:*`- # filters.- def maybe_add_group_conversion_rate(q, site, query) do- if :group_conversion_rate in query.metrics do- group_totals_query =- query- |> Query.remove_filters(["event:goal", "event:props"])- |> Query.set_metrics([:visitors])- |> Query.set_order_by([])-- from(e in subquery(q),- left_join: c in subquery(build(group_totals_query, site)),- on: ^build_group_by_join(query)- )- |> select_merge_as([e, c], %{- total_visitors: c.visitors,- group_conversion_rate:- fragment(- "if(? > 0, round(? / ? * 100, 1), 0)",- c.visitors,- e.visitors,- c.visitors- )- })- |> select_join_fields(query, query.dimensions, e)- |> select_join_fields(query, List.delete(query.metrics, :group_conversion_rate), e)- else- q- end- end-defp join_query_results({nil, _}, {nil, _}), do: nildefp join_query_results({events_q, events_query}, {nil, _}),diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.exnew file mode 100644index 0000000000..c46a5f1d7b--- /dev/null+++ b/lib/plausible/stats/sql/special_metrics.ex@@ -0,0 +1,152 @@+defmodule Plausible.Stats.SQL.SpecialMetrics do+ @moduledoc """+ This module defines how special metrics like `conversion_rate` and+ `percentage` are calculated.+ """++ use Plausible.Stats.SQL.Fragments++ alias Plausible.Stats.{Base, Query, SQL}++ import Ecto.Query+ import Plausible.Stats.Util++ def add(q, site, query) do+ q+ |> maybe_add_percentage_metric(site, query)+ |> maybe_add_global_conversion_rate(site, query)+ |> maybe_add_group_conversion_rate(site, query)+ end++ defp maybe_add_percentage_metric(q, site, query) do+ if :percentage in query.metrics do+ total_query =+ Query.set(query,+ dimensions: [],+ include_imported: query.include_imported+ )++ q+ |> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported))+ |> select_merge_as([], %{+ percentage:+ fragment(+ "if(? > 0, round(? / ? * 100, 1), null)",+ selected_as(:total_visitors),+ selected_as(:visitors),+ selected_as(:total_visitors)+ )+ })+ else+ q+ end+ end++ # Adds conversion_rate metric to query, calculated as+ # X / Y where Y is the same breakdown value without goal or props+ # filters.+ def maybe_add_global_conversion_rate(q, site, query) do+ if :conversion_rate in query.metrics do+ total_query =+ query+ |> Query.remove_filters(["event:goal", "event:props"])+ |> Query.set(+ dimensions: [],+ include_imported: query.include_imported+ )++ q+ |> select_merge_as(+ [],+ total_visitors_subquery(site, total_query, query.include_imported)+ )+ |> select_merge_as([e], %{+ conversion_rate:+ fragment(+ "if(? > 0, round(? / ? * 100, 1), 0)",+ selected_as(:total_visitors),+ selected_as(:visitors),+ selected_as(:total_visitors)+ )+ })+ else+ q+ end+ end++ # This function injects a group_conversion_rate metric into+ # a dimensional query. It is calculated as X / Y, where:+ #+ # * X is the number of conversions for a set of dimensions+ # result (conversion = number of visitors who+ # completed the filtered goal with the filtered+ # custom properties).+ #+ # * Y is the number of all visitors for this set of dimensions+ # result without the `event:goal` and `event:props:*`+ # filters.+ def maybe_add_group_conversion_rate(q, site, query) do+ if :group_conversion_rate in query.metrics do+ group_totals_query =+ query+ |> Query.remove_filters(["event:goal", "event:props"])+ |> Query.set(+ metrics: [:visitors],+ order_by: [],+ include_imported: query.include_imported+ )++ from(e in subquery(q),+ left_join: c in subquery(SQL.QueryBuilder.build(group_totals_query, site)),+ on: ^SQL.QueryBuilder.build_group_by_join(query)+ )+ |> select_merge_as([e, c], %{+ total_visitors: c.visitors,+ group_conversion_rate:+ fragment(+ "if(? > 0, round(? / ? * 100, 1), 0)",+ c.visitors,+ e.visitors,+ c.visitors+ )+ })+ |> select_join_fields(query, query.dimensions, e)+ |> select_join_fields(query, List.delete(query.metrics, :group_conversion_rate), e)+ else+ q+ end+ end++ defp total_visitors(site, query) do+ Base.base_event_query(site, query)+ |> select([e],+ total_visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)+ )+ end++ # `total_visitors_subquery` returns a subquery which selects `total_visitors` -+ # the number used as the denominator in the calculation of `conversion_rate` and+ # `percentage` metrics.++ # Usually, when calculating the totals, a new query is passed into this function,+ # where certain filters (e.g. goal, props) are removed. That might make the query+ # able to include imported data. However, we always want to include imported data+ # only if it's included in the base query - otherwise the total will be based on+ # a different data set, making the metric inaccurate. This is why we're using an+ # explicit `include_imported` argument here.+ defp total_visitors_subquery(site, query, include_imported)++ defp total_visitors_subquery(site, query, true = _include_imported) do+ wrap_alias([], %{+ total_visitors:+ subquery(total_visitors(site, query)) ++ subquery(Plausible.Stats.Imported.total_imported_visitors(site, query))+ })+ end++ defp total_visitors_subquery(site, query, false = _include_imported) do+ wrap_alias([], %{+ total_visitors: subquery(total_visitors(site, query))+ })+ end+enddiff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.exindex 71bcdad495..73eeecaeef 100644--- a/lib/plausible/stats/sql/where_builder.ex+++ b/lib/plausible/stats/sql/where_builder.ex@@ -51,7 +51,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do)end- defp filter_site_time_range(:sessions, site, %Query{experimental_session_count?: true} = query) do+ defp filter_site_time_range(:sessions, site, query) do{first_datetime, last_datetime} = utc_boundaries(query, site)# Counts each _active_ session in time range even if they started before@@ -61,15 +61,6 @@ defmodule Plausible.Stats.SQL.WhereBuilder do)end- defp filter_site_time_range(:sessions, site, query) do- {first_datetime, last_datetime} = utc_boundaries(query, site)-- dynamic(- [s],- s.site_id == ^site.id and s.start >= ^first_datetime and s.start < ^last_datetime- )- end-defp add_filter(:events, _query, [:is, "event:name", list]) dodynamic([e], e.name in ^list)enddiff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.exnew file mode 100644index 0000000000..c8c0d7a71f--- /dev/null+++ b/lib/plausible/stats/time.ex@@ -0,0 +1,118 @@+defmodule Plausible.Stats.Time do+ @moduledoc """+ Collection of functions to work with time in queries.+ """++ alias Plausible.Stats.Query+ def format_datetime(%Date{} = date), do: Date.to_string(date)++ def format_datetime(%DateTime{} = datetime),+ do: Timex.format!(datetime, "{YYYY}-{0M}-{0D} {h24}:{m}:{s}")++ # Realtime graphs return numbers+ def format_datetime(other), do: other++ @doc """+ Returns list of time bucket labels for the given query.+ """+ def time_dimension(query) do+ Enum.find(query.dimensions, &String.starts_with?(&1, "time"))+ end++ def time_labels(query) do+ time_labels_for_dimension(time_dimension(query), query)+ end++ defp time_labels_for_dimension("time:month", query) do+ n_buckets =+ Timex.diff(+ query.date_range.last,+ Date.beginning_of_month(query.date_range.first),+ :months+ )++ Enum.map(n_buckets..0, fn shift ->+ query.date_range.last+ |> Date.beginning_of_month()+ |> Timex.shift(months: -shift)+ |> format_datetime()+ end)+ end++ defp time_labels_for_dimension("time:week", query) do+ n_buckets =+ Timex.diff(+ query.date_range.last,+ Date.beginning_of_week(query.date_range.first),+ :weeks+ )++ Enum.map(0..n_buckets, fn shift ->+ query.date_range.first+ |> Timex.shift(weeks: shift)+ |> date_or_weekstart(query)+ |> format_datetime()+ end)+ end++ defp time_labels_for_dimension("time:day", query) do+ query.date_range+ |> Enum.into([])+ |> Enum.map(&format_datetime/1)+ end++ @full_day_in_hours 23+ defp time_labels_for_dimension("time:hour", query) do+ n_buckets =+ if query.date_range.first == query.date_range.last do+ @full_day_in_hours+ else+ end_time =+ query.date_range.last+ |> Timex.to_datetime()+ |> Timex.end_of_day()++ Timex.diff(end_time, query.date_range.first, :hours)+ end++ Enum.map(0..n_buckets, fn step ->+ query.date_range.first+ |> Timex.to_datetime()+ |> Timex.shift(hours: step)+ |> DateTime.truncate(:second)+ |> format_datetime()+ end)+ end++ # Only supported in dashboards not via API+ defp time_labels_for_dimension("time:minute", %Query{period: "30m"}) do+ Enum.into(-30..-1, [])+ end++ @full_day_in_minutes 24 * 60 - 1+ defp time_labels_for_dimension("time:minute", query) do+ n_buckets =+ if query.date_range.first == query.date_range.last do+ @full_day_in_minutes+ else+ Timex.diff(query.date_range.last, query.date_range.first, :minutes)+ end++ Enum.map(0..n_buckets, fn step ->+ query.date_range.first+ |> Timex.to_datetime()+ |> Timex.shift(minutes: step)+ |> format_datetime()+ end)+ end++ defp date_or_weekstart(date, query) do+ weekstart = Timex.beginning_of_week(date)++ if Enum.member?(query.date_range, weekstart) do+ weekstart+ else+ date+ end+ end+enddiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 4914023bfc..5fc3938ed0 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -1,292 +1,70 @@defmodule Plausible.Stats.Timeseries do- use Plausible.ClickhouseRepouse Plausible- alias Plausible.Stats.{Query, Util, Imported}- import Plausible.Stats.{Base}- import Ecto.Query- use Plausible.Stats.SQL.Fragments+ use Plausible.ClickhouseRepo+ alias Plausible.Stats.{Query, QueryOptimizer, QueryResult, SQL}- @typep metric ::- :pageviews- | :events- | :visitors- | :visits- | :bounce_rate- | :visit_duration- | :average_revenue- | :total_revenue- @typep value :: nil | integer() | float()- @type results :: nonempty_list(%{required(:date) => Date.t(), required(metric()) => value()})+ @time_dimension %{+ "month" => "time:month",+ "week" => "time:week",+ "date" => "time:day",+ "hour" => "time:hour",+ "minute" => "time:minute"+ }def timeseries(site, query, metrics) do- steps = buckets(query)-- {event_metrics, session_metrics, _} =- Plausible.Stats.TableDecider.partition_metrics(metrics, query)-- {currency, event_metrics} =+ {currency, metrics} =on_ee do- Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, event_metrics)+ Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, metrics)else- {nil, event_metrics}+ {nil, metrics}end- Query.trace(query, metrics)-- [event_result, session_result] =- Plausible.ClickhouseRepo.parallel_tasks([- fn -> events_timeseries(site, query, event_metrics) end,- fn -> sessions_timeseries(site, query, session_metrics) end- ])-- Enum.map(steps, fn step ->- empty_row(step, metrics)- |> Map.merge(Enum.find(event_result, fn row -> date_eq(row[:date], step) end) || %{})- |> Map.merge(Enum.find(session_result, fn row -> date_eq(row[:date], step) end) || %{})- |> Map.update!(:date, &date_format/1)- |> cast_revenue_metrics_to_money(currency)- end)- |> Util.keep_requested_metrics(metrics)- end-- defp events_timeseries(_, _, []), do: []-- defp events_timeseries(site, query, metrics) do- metrics = Util.maybe_add_visitors_metric(metrics)-- from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))- |> select_bucket(:events, site, query)- |> Imported.merge_imported_timeseries(site, query, metrics)- |> maybe_add_timeseries_conversion_rate(site, query, metrics)- |> ClickhouseRepo.all()- end+ query_with_metrics =+ Query.set(+ query,+ metrics: transform_metrics(metrics, %{conversion_rate: :group_conversion_rate}),+ dimensions: [time_dimension(query)],+ order_by: [{time_dimension(query), :asc}],+ v2: true,+ include: %{time_labels: true, imports: query.include.imports}+ )+ |> QueryOptimizer.optimize()- defp sessions_timeseries(_, _, []), do: []+ q = SQL.QueryBuilder.build(query_with_metrics, site)- defp sessions_timeseries(site, query, metrics) do- from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))- |> filter_converted_sessions(site, query)- |> select_bucket(:sessions, site, query)- |> Imported.merge_imported_timeseries(site, query, metrics)+ q|> ClickhouseRepo.all()- |> Util.keep_requested_metrics(metrics)- end-- defp buckets(%Query{interval: "month"} = query) do- n_buckets = Timex.diff(query.date_range.last, query.date_range.first, :months)-- Enum.map(n_buckets..0, fn shift ->- query.date_range.last- |> Timex.beginning_of_month()- |> Timex.shift(months: -shift)- end)- end-- defp buckets(%Query{interval: "week"} = query) do- n_buckets = Timex.diff(query.date_range.last, query.date_range.first, :weeks)-- Enum.map(0..n_buckets, fn shift ->- query.date_range.first- |> Timex.shift(weeks: shift)- |> date_or_weekstart(query)- end)+ |> QueryResult.from(query_with_metrics)+ |> build_timeseries_result(query_with_metrics, currency)+ |> transform_keys(%{group_conversion_rate: :conversion_rate})end- defp buckets(%Query{interval: "date"} = query) do- Enum.into(query.date_range, [])- end+ defp time_dimension(query), do: Map.fetch!(@time_dimension, query.interval)- @full_day_in_hours 23- defp buckets(%Query{interval: "hour"} = query) do- n_buckets =- if query.date_range.first == query.date_range.last do- @full_day_in_hours- else- Timex.diff(query.date_range.last, query.date_range.first, :hours)- end+ defp build_timeseries_result(query_result, query, currency) do+ results_map =+ query_result.results+ |> Enum.map(fn %{dimensions: [time_dimension_value], metrics: entry_metrics} ->+ metrics_map = Enum.zip(query.metrics, entry_metrics) |> Enum.into(%{})- Enum.map(0..n_buckets, fn step ->- query.date_range.first- |> Timex.to_datetime()- |> Timex.shift(hours: step)- end)- end-- defp buckets(%Query{period: "30m", interval: "minute"}) do- Enum.into(-30..-1, [])- end-- @full_day_in_minutes 1439- defp buckets(%Query{interval: "minute"} = query) do- n_buckets =- if query.date_range.first == query.date_range.last do- @full_day_in_minutes- else- Timex.diff(query.date_range.last, query.date_range.first, :minutes)- end-- Enum.map(0..n_buckets, fn step ->- query.date_range.first- |> Timex.to_datetime()- |> Timex.shift(minutes: step)+ {+ time_dimension_value,+ Map.put(metrics_map, :date, time_dimension_value)+ }+ end)+ |> Enum.into(%{})++ query_result.meta.time_labels+ |> Enum.map(fn key ->+ Map.get(+ results_map,+ key,+ empty_row(key, query.metrics)+ )+ |> cast_revenue_metrics_to_money(currency)end)end- defp date_eq(%DateTime{} = left, %DateTime{} = right) do- NaiveDateTime.compare(left, right) == :eq- end-- defp date_eq(%Date{} = left, %Date{} = right) do- Date.compare(left, right) == :eq- end-- defp date_eq(left, right) do- left == right- end-- defp date_format(%DateTime{} = date) do- Timex.format!(date, "{YYYY}-{0M}-{0D} {h24}:{m}:{s}")- end-- defp date_format(date) do- date- end-- defp select_bucket(q, _table, site, %Query{interval: "month"}) do- from(- e in q,- group_by: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone),- order_by: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone),- select_merge: %{- date: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone)- }- )- end-- defp select_bucket(q, _table, site, %Query{interval: "week"} = query) do- {first_datetime, _} = utc_boundaries(query, site)-- from(- e in q,- select_merge: %{date: weekstart_not_before(e.timestamp, ^first_datetime, ^site.timezone)},- group_by: weekstart_not_before(e.timestamp, ^first_datetime, ^site.timezone),- order_by: weekstart_not_before(e.timestamp, ^first_datetime, ^site.timezone)- )- end-- defp select_bucket(q, _table, site, %Query{interval: "date"}) do- from(- e in q,- group_by: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone),- order_by: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone),- select_merge: %{- date: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone)- }- )- end-- defp select_bucket(q, :sessions, site, %Query{- interval: "hour",- experimental_session_count?: true- }) do- bucket_with_timeslots(q, site, 3600)- end-- defp select_bucket(q, _table, site, %Query{interval: "hour"}) do- from(- e in q,- group_by: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone),- order_by: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone),- select_merge: %{- date: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone)- }- )- end-- defp select_bucket(q, :sessions, _site, %Query{- interval: "minute",- period: "30m",- experimental_session_count?: true- }) do- from(- s in q,- array_join:- bucket in fragment(- "timeSlots(?, toUInt32(timeDiff(?, ?)), ?)",- s.start,- s.start,- s.timestamp,- 60- ),- group_by: fragment("dateDiff('minute', now(), ?)", bucket),- order_by: fragment("dateDiff('minute', now(), ?)", bucket),- select_merge: %{- date: fragment("dateDiff('minute', now(), ?)", bucket)- }- )- end-- defp select_bucket(q, _table, _site, %Query{interval: "minute", period: "30m"}) do- from(- e in q,- group_by: fragment("dateDiff('minute', now(), ?)", e.timestamp),- order_by: fragment("dateDiff('minute', now(), ?)", e.timestamp),- select_merge: %{- date: fragment("dateDiff('minute', now(), ?)", e.timestamp)- }- )- end-- defp select_bucket(q, _table, site, %Query{interval: "minute"}) do- from(- e in q,- group_by: fragment("toStartOfMinute(toTimeZone(?, ?))", e.timestamp, ^site.timezone),- order_by: fragment("toStartOfMinute(toTimeZone(?, ?))", e.timestamp, ^site.timezone),- select_merge: %{- date: fragment("toStartOfMinute(toTimeZone(?, ?))", e.timestamp, ^site.timezone)- }- )- end-- defp select_bucket(q, :sessions, site, %Query{- interval: "minute",- experimental_session_count?: true- }) do- bucket_with_timeslots(q, site, 60)- end-- # Includes session in _every_ time bucket it was active in.- # Only done in hourly and minute graphs for performance reasons.- defp bucket_with_timeslots(q, site, period_in_seconds) do- from(- s in q,- array_join:- bucket in fragment(- "timeSlots(toTimeZone(?, ?), toUInt32(timeDiff(?, ?)), toUInt32(?))",- s.start,- ^site.timezone,- s.start,- s.timestamp,- ^period_in_seconds- ),- group_by: bucket,- order_by: bucket,- select_merge: %{- date: fragment("?", bucket)- }- )- end-- defp date_or_weekstart(date, query) do- weekstart = Timex.beginning_of_week(date)-- if Enum.member?(query.date_range, weekstart) do- weekstart- else- date- end- end-defp empty_row(date, metrics) doEnum.reduce(metrics, %{date: date}, fn metric, row ->case metric do@@ -296,7 +74,8 @@ defmodule Plausible.Stats.Timeseries do:visits -> Map.merge(row, %{visits: 0}):views_per_visit -> Map.merge(row, %{views_per_visit: 0.0}):conversion_rate -> Map.merge(row, %{conversion_rate: 0.0})- :bounce_rate -> Map.merge(row, %{bounce_rate: nil})+ :group_conversion_rate -> Map.merge(row, %{group_conversion_rate: 0.0})+ :bounce_rate -> Map.merge(row, %{bounce_rate: 0.0}):visit_duration -> Map.merge(row, %{visit_duration: nil}):average_revenue -> Map.merge(row, %{average_revenue: nil}):total_revenue -> Map.merge(row, %{total_revenue: nil})@@ -304,6 +83,19 @@ defmodule Plausible.Stats.Timeseries doend)end+ defp transform_metrics(metrics, to_replace) do+ Enum.map(metrics, &Map.get(to_replace, &1, &1))+ end++ defp transform_keys(results, keys_to_replace) do+ Enum.map(results, fn map ->+ Enum.map(map, fn {key, val} ->+ {Map.get(keys_to_replace, key, key), val}+ end)+ |> Enum.into(%{})+ end)+ end+on_ee dodefp cast_revenue_metrics_to_money(results, revenue_goals) doPlausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals)@@ -311,41 +103,4 @@ defmodule Plausible.Stats.Timeseries doelsedefp cast_revenue_metrics_to_money(results, _revenue_goals), do: resultsend-- defp maybe_add_timeseries_conversion_rate(q, site, query, metrics) do- if :conversion_rate in metrics do- # Having removed some filters, the query might become eligible- # for including imported data. However, we still want to make- # sure that that include_imported is in sync between original- # and the totals query.- totals_query =- query- |> Query.remove_filters(["event:goal", "event:props"])- |> struct!(include_imported: query.include_imported)-- totals_timeseries_q =- from(e in base_event_query(site, totals_query),- select: ^select_event_metrics([:visitors])- )- |> select_bucket(:events, site, totals_query)- |> Imported.merge_imported_timeseries(site, totals_query, [:visitors])-- from(e in subquery(q),- left_join: c in subquery(totals_timeseries_q),- on: e.date == c.date,- select_merge: %{- total_visitors: c.visitors,- conversion_rate:- fragment(- "if(? > 0, round(? / ? * 100, 1), 0)",- c.visitors,- e.visitors,- c.visitors- )- }- )- else- q- end- endenddiff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.exindex de1411f637..65e95078dd 100644--- a/lib/plausible/stats/util.ex+++ b/lib/plausible/stats/util.ex@@ -6,7 +6,6 @@ defmodule Plausible.Stats.Util do@manually_removable_metrics [:__internal_visits,:visitors,- :__breakdown_value,:total_visitors]diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 5dcf7800cf..e5f01eda35 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -174,8 +174,10 @@ defmodule PlausibleWeb.Api.StatsController dodefp build_full_intervals(%{interval: "week", date_range: range}, labels) dofor label <- labels, into: %{} do- interval_start = Timex.beginning_of_week(label)- interval_end = Timex.end_of_week(label)+ date = Date.from_iso8601!(label)++ interval_start = Timex.beginning_of_week(date)+ interval_end = Timex.end_of_week(date)within_interval? = Enum.member?(range, interval_start) && Enum.member?(range, interval_end)@@ -185,8 +187,10 @@ defmodule PlausibleWeb.Api.StatsController dodefp build_full_intervals(%{interval: "month", date_range: range}, labels) dofor label <- labels, into: %{} do- interval_start = Timex.beginning_of_month(label)- interval_end = Timex.end_of_month(label)+ date = Date.from_iso8601!(label)++ interval_start = Timex.beginning_of_month(date)+ interval_end = Timex.end_of_month(date)within_interval? = Enum.member?(range, interval_start) && Enum.member?(range, interval_end)diff --git a/test/plausible/stats/query_optimizer_test.exs b/test/plausible/stats/query_optimizer_test.exsindex aee73b88c0..d35b958bcd 100644--- a/test/plausible/stats/query_optimizer_test.exs+++ b/test/plausible/stats/query_optimizer_test.exs@@ -74,17 +74,17 @@ defmodule Plausible.Stats.QueryOptimizerTest doassert perform(%{date_range: Date.range(~D[2022-01-01], ~D[2022-02-16]),dimensions: ["time"]- }).dimensions == ["time:month"]+ }).dimensions == ["time:week"]assert perform(%{date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]),dimensions: ["time"]- }).dimensions == ["time:month"]+ }).dimensions == ["time:week"]assert perform(%{date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]),dimensions: ["time"]- }).dimensions == ["time:month"]+ }).dimensions == ["time:week"]assert perform(%{date_range: Date.range(~D[2022-01-01], ~D[2023-11-16]),diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex bf3f3bedb4..017ba51575 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -27,7 +27,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -47,7 +47,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -85,7 +85,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -115,7 +115,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -160,7 +160,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -184,7 +184,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -209,7 +209,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -248,20 +248,21 @@ defmodule Plausible.Stats.Filters.QueryParserTest doenddescribe "include validation" do- test "setting include.imports", %{site: site} do+ test "setting include values", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all",- "include" => %{"imports" => true}+ "dimensions" => ["time"],+ "include" => %{"imports" => true, "time_labels" => true}}|> check_success(site, %{metrics: [:visitors],date_range: @date_range,filters: [],- dimensions: [],+ dimensions: ["time"],order_by: nil,timezone: site.timezone,- imported_data_requested: true,+ include: %{imports: true, time_labels: true},preloaded_goals: []})end@@ -274,6 +275,15 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_error(site, ~r/Invalid include passed/)end++ test "setting include.time_labels without time dimension", %{site: site} do+ %{+ "metrics" => ["visitors"],+ "date_range" => "all",+ "include" => %{"time_labels" => true}+ }+ |> check_error(site, ~r/Invalid include.time_labels: requires a time dimension/)+ endenddescribe "event:goal filter validation" do@@ -297,7 +307,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: [{:page, "/thank-you"}, {:event, "Signup"}]})end@@ -379,7 +389,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:#{unquote(dimension)}"],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -399,7 +409,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["visit:#{unquote(dimension)}"],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -418,7 +428,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:props:foobar"],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -474,7 +484,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: [{:events, :desc}, {:visitors, :asc}],timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -493,7 +503,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:name"],order_by: [{"event:name", :desc}],timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -589,7 +599,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: [event: "Signup"]})end@@ -609,7 +619,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:goal"],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: [event: "Signup"]})end@@ -631,7 +641,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: [event: "Signup"]})end@@ -675,7 +685,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["visit:device"],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -705,7 +715,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:page"],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})end@@ -723,7 +733,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false,+ include: %{imports: false, time_labels: false},preloaded_goals: []})enddiff --git a/test/plausible/stats/time_test.exs b/test/plausible/stats/time_test.exsnew file mode 100644index 0000000000..e368a0cbc5--- /dev/null+++ b/test/plausible/stats/time_test.exs@@ -0,0 +1,157 @@+defmodule Plausible.Stats.TimeTest do+ use Plausible.DataCase, async: true++ import Plausible.Stats.Time++ describe "time_labels/1" do+ test "with time:month dimension" do+ assert time_labels(%{+ dimensions: ["visit:device", "time:month"],+ date_range: Date.range(~D[2022-01-17], ~D[2022-02-01])+ }) == [+ "2022-01-01",+ "2022-02-01"+ ]++ assert time_labels(%{+ dimensions: ["visit:device", "time:month"],+ date_range: Date.range(~D[2022-01-01], ~D[2022-03-07])+ }) == [+ "2022-01-01",+ "2022-02-01",+ "2022-03-01"+ ]+ end++ test "with time:week dimension" do+ assert time_labels(%{+ dimensions: ["time:week"],+ date_range: Date.range(~D[2020-12-20], ~D[2021-01-08])+ }) == [+ "2020-12-20",+ "2020-12-21",+ "2020-12-28",+ "2021-01-04"+ ]++ assert time_labels(%{+ dimensions: ["time:week"],+ date_range: Date.range(~D[2020-12-21], ~D[2021-01-03])+ }) == [+ "2020-12-21",+ "2020-12-28"+ ]+ end++ test "with time:day dimension" do+ assert time_labels(%{+ dimensions: ["time:day"],+ date_range: Date.range(~D[2022-01-17], ~D[2022-02-02])+ }) == [+ "2022-01-17",+ "2022-01-18",+ "2022-01-19",+ "2022-01-20",+ "2022-01-21",+ "2022-01-22",+ "2022-01-23",+ "2022-01-24",+ "2022-01-25",+ "2022-01-26",+ "2022-01-27",+ "2022-01-28",+ "2022-01-29",+ "2022-01-30",+ "2022-01-31",+ "2022-02-01",+ "2022-02-02"+ ]+ end++ test "with time:hour dimension" do+ assert time_labels(%{+ dimensions: ["time:hour"],+ date_range: Date.range(~D[2022-01-17], ~D[2022-01-17])+ }) == [+ "2022-01-17 00:00:00",+ "2022-01-17 01:00:00",+ "2022-01-17 02:00:00",+ "2022-01-17 03:00:00",+ "2022-01-17 04:00:00",+ "2022-01-17 05:00:00",+ "2022-01-17 06:00:00",+ "2022-01-17 07:00:00",+ "2022-01-17 08:00:00",+ "2022-01-17 09:00:00",+ "2022-01-17 10:00:00",+ "2022-01-17 11:00:00",+ "2022-01-17 12:00:00",+ "2022-01-17 13:00:00",+ "2022-01-17 14:00:00",+ "2022-01-17 15:00:00",+ "2022-01-17 16:00:00",+ "2022-01-17 17:00:00",+ "2022-01-17 18:00:00",+ "2022-01-17 19:00:00",+ "2022-01-17 20:00:00",+ "2022-01-17 21:00:00",+ "2022-01-17 22:00:00",+ "2022-01-17 23:00:00"+ ]++ assert time_labels(%{+ dimensions: ["time:hour"],+ date_range: Date.range(~D[2022-01-17], ~D[2022-01-18])+ }) == [+ "2022-01-17 00:00:00",+ "2022-01-17 01:00:00",+ "2022-01-17 02:00:00",+ "2022-01-17 03:00:00",+ "2022-01-17 04:00:00",+ "2022-01-17 05:00:00",+ "2022-01-17 06:00:00",+ "2022-01-17 07:00:00",+ "2022-01-17 08:00:00",+ "2022-01-17 09:00:00",+ "2022-01-17 10:00:00",+ "2022-01-17 11:00:00",+ "2022-01-17 12:00:00",+ "2022-01-17 13:00:00",+ "2022-01-17 14:00:00",+ "2022-01-17 15:00:00",+ "2022-01-17 16:00:00",+ "2022-01-17 17:00:00",+ "2022-01-17 18:00:00",+ "2022-01-17 19:00:00",+ "2022-01-17 20:00:00",+ "2022-01-17 21:00:00",+ "2022-01-17 22:00:00",+ "2022-01-17 23:00:00",+ "2022-01-18 00:00:00",+ "2022-01-18 01:00:00",+ "2022-01-18 02:00:00",+ "2022-01-18 03:00:00",+ "2022-01-18 04:00:00",+ "2022-01-18 05:00:00",+ "2022-01-18 06:00:00",+ "2022-01-18 07:00:00",+ "2022-01-18 08:00:00",+ "2022-01-18 09:00:00",+ "2022-01-18 10:00:00",+ "2022-01-18 11:00:00",+ "2022-01-18 12:00:00",+ "2022-01-18 13:00:00",+ "2022-01-18 14:00:00",+ "2022-01-18 15:00:00",+ "2022-01-18 16:00:00",+ "2022-01-18 17:00:00",+ "2022-01-18 18:00:00",+ "2022-01-18 19:00:00",+ "2022-01-18 20:00:00",+ "2022-01-18 21:00:00",+ "2022-01-18 22:00:00",+ "2022-01-18 23:00:00"+ ]+ end+ end+enddiff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csvindex a49423c670..fa82414a60 100644--- a/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv@@ -1,32 +1,32 @@date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration-2021-09-20,0,0,0,0.0,,-2021-09-21,0,0,0,0.0,,-2021-09-22,0,0,0,0.0,,-2021-09-23,0,0,0,0.0,,-2021-09-24,0,0,0,0.0,,-2021-09-25,0,0,0,0.0,,-2021-09-26,0,0,0,0.0,,-2021-09-27,0,0,0,0.0,,-2021-09-28,0,0,0,0.0,,-2021-09-29,0,0,0,0.0,,-2021-09-30,0,0,0,0.0,,-2021-10-01,0,0,0,0.0,,-2021-10-02,0,0,0,0.0,,-2021-10-03,0,0,0,0.0,,-2021-10-04,0,0,0,0.0,,-2021-10-05,0,0,0,0.0,,-2021-10-06,0,0,0,0.0,,-2021-10-07,0,0,0,0.0,,-2021-10-08,0,0,0,0.0,,-2021-10-09,0,0,0,0.0,,-2021-10-10,0,0,0,0.0,,-2021-10-11,0,0,0,0.0,,-2021-10-12,0,0,0,0.0,,-2021-10-13,0,0,0,0.0,,-2021-10-14,0,0,0,0.0,,-2021-10-15,0,0,0,0.0,,-2021-10-16,0,0,0,0.0,,-2021-10-17,0,0,0,0.0,,-2021-10-18,0,0,0,0.0,,-2021-10-19,0,0,0,0.0,,+2021-09-20,0,0,0,0.0,0.0,+2021-09-21,0,0,0,0.0,0.0,+2021-09-22,0,0,0,0.0,0.0,+2021-09-23,0,0,0,0.0,0.0,+2021-09-24,0,0,0,0.0,0.0,+2021-09-25,0,0,0,0.0,0.0,+2021-09-26,0,0,0,0.0,0.0,+2021-09-27,0,0,0,0.0,0.0,+2021-09-28,0,0,0,0.0,0.0,+2021-09-29,0,0,0,0.0,0.0,+2021-09-30,0,0,0,0.0,0.0,+2021-10-01,0,0,0,0.0,0.0,+2021-10-02,0,0,0,0.0,0.0,+2021-10-03,0,0,0,0.0,0.0,+2021-10-04,0,0,0,0.0,0.0,+2021-10-05,0,0,0,0.0,0.0,+2021-10-06,0,0,0,0.0,0.0,+2021-10-07,0,0,0,0.0,0.0,+2021-10-08,0,0,0,0.0,0.0,+2021-10-09,0,0,0,0.0,0.0,+2021-10-10,0,0,0,0.0,0.0,+2021-10-11,0,0,0,0.0,0.0,+2021-10-12,0,0,0,0.0,0.0,+2021-10-13,0,0,0,0.0,0.0,+2021-10-14,0,0,0,0.0,0.0,+2021-10-15,0,0,0,0.0,0.0,+2021-10-16,0,0,0,0.0,0.0,+2021-10-17,0,0,0,0.0,0.0,+2021-10-18,0,0,0,0.0,0.0,+2021-10-19,0,0,0,0.0,0.0,2021-10-20,1,1,1,2.0,0,60diff --git a/test/plausible_web/controllers/CSVs/30d/visitors.csv b/test/plausible_web/controllers/CSVs/30d/visitors.csvindex 6bac77976e..76dbc7a9c5 100644--- a/test/plausible_web/controllers/CSVs/30d/visitors.csv+++ b/test/plausible_web/controllers/CSVs/30d/visitors.csv@@ -1,32 +1,32 @@date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration2021-09-20,1,1,1,1.0,100,0-2021-09-21,0,0,0,0.0,,-2021-09-22,0,0,0,0.0,,-2021-09-23,0,0,0,0.0,,-2021-09-24,0,0,0,0.0,,-2021-09-25,0,0,0,0.0,,-2021-09-26,0,0,0,0.0,,-2021-09-27,0,0,0,0.0,,-2021-09-28,0,0,0,0.0,,-2021-09-29,0,0,0,0.0,,-2021-09-30,0,0,0,0.0,,-2021-10-01,0,0,0,0.0,,-2021-10-02,0,0,0,0.0,,-2021-10-03,0,0,0,0.0,,-2021-10-04,0,0,0,0.0,,-2021-10-05,0,0,0,0.0,,-2021-10-06,0,0,0,0.0,,-2021-10-07,0,0,0,0.0,,-2021-10-08,0,0,0,0.0,,-2021-10-09,0,0,0,0.0,,-2021-10-10,0,0,0,0.0,,-2021-10-11,0,0,0,0.0,,-2021-10-12,0,0,0,0.0,,-2021-10-13,0,0,0,0.0,,-2021-10-14,0,0,0,0.0,,-2021-10-15,0,0,0,0.0,,-2021-10-16,0,0,0,0.0,,-2021-10-17,0,0,0,0.0,,-2021-10-18,0,0,0,0.0,,+2021-09-21,0,0,0,0.0,0.0,+2021-09-22,0,0,0,0.0,0.0,+2021-09-23,0,0,0,0.0,0.0,+2021-09-24,0,0,0,0.0,0.0,+2021-09-25,0,0,0,0.0,0.0,+2021-09-26,0,0,0,0.0,0.0,+2021-09-27,0,0,0,0.0,0.0,+2021-09-28,0,0,0,0.0,0.0,+2021-09-29,0,0,0,0.0,0.0,+2021-09-30,0,0,0,0.0,0.0,+2021-10-01,0,0,0,0.0,0.0,+2021-10-02,0,0,0,0.0,0.0,+2021-10-03,0,0,0,0.0,0.0,+2021-10-04,0,0,0,0.0,0.0,+2021-10-05,0,0,0,0.0,0.0,+2021-10-06,0,0,0,0.0,0.0,+2021-10-07,0,0,0,0.0,0.0,+2021-10-08,0,0,0,0.0,0.0,+2021-10-09,0,0,0,0.0,0.0,+2021-10-10,0,0,0,0.0,0.0,+2021-10-11,0,0,0,0.0,0.0,+2021-10-12,0,0,0,0.0,0.0,+2021-10-13,0,0,0,0.0,0.0,+2021-10-14,0,0,0,0.0,0.0,+2021-10-15,0,0,0,0.0,0.0,+2021-10-16,0,0,0,0.0,0.0,+2021-10-17,0,0,0,0.0,0.0,+2021-10-18,0,0,0,0.0,0.0,2021-10-19,2,2,2,1.0,50,302021-10-20,1,2,1,2.0,0,60diff --git a/test/plausible_web/controllers/CSVs/6m/visitors.csv b/test/plausible_web/controllers/CSVs/6m/visitors.csvindex 230cc9d82c..6858d4466e 100644--- a/test/plausible_web/controllers/CSVs/6m/visitors.csv+++ b/test/plausible_web/controllers/CSVs/6m/visitors.csv@@ -1,7 +1,7 @@date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration2021-05-01,1,1,1,1.0,100,0-2021-06-01,0,0,0,0.0,,-2021-07-01,0,0,0,0.0,,-2021-08-01,0,0,0,0.0,,+2021-06-01,0,0,0,0.0,0.0,+2021-07-01,0,0,0,0.0,0.0,+2021-08-01,0,0,0,0.0,0.0,2021-09-01,1,1,1,1.0,100,02021-10-01,3,4,3,1.33,33,40diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex 331687e1f0..9454196398 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -1096,7 +1096,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doenddescribe "timeseries" do- test "shows hourly data for a certain date", %{conn: conn, site: site} do+ test "shows hourly data for a certain date with time_labels", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:10:00]),@@ -1108,16 +1108,44 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"site_id" => site.domain,"metrics" => ["visitors", "pageviews", "visits", "visit_duration", "bounce_rate"],"date_range" => ["2021-01-01", "2021-01-01"],- "dimensions" => ["time:hour"]+ "dimensions" => ["time:hour"],+ "include" => %{"time_labels" => true}})assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["2021-01-01T00:00:00Z"], "metrics" => [1, 2, 1, 600, 0]},- %{"dimensions" => ["2021-01-01T23:00:00Z"], "metrics" => [1, 1, 1, 0, 100]}+ %{"dimensions" => ["2021-01-01 00:00:00"], "metrics" => [1, 2, 1, 600, 0]},+ %{"dimensions" => ["2021-01-01 23:00:00"], "metrics" => [1, 1, 1, 0, 100]}+ ]++ assert json_response(conn, 200)["meta"]["time_labels"] == [+ "2021-01-01 00:00:00",+ "2021-01-01 01:00:00",+ "2021-01-01 02:00:00",+ "2021-01-01 03:00:00",+ "2021-01-01 04:00:00",+ "2021-01-01 05:00:00",+ "2021-01-01 06:00:00",+ "2021-01-01 07:00:00",+ "2021-01-01 08:00:00",+ "2021-01-01 09:00:00",+ "2021-01-01 10:00:00",+ "2021-01-01 11:00:00",+ "2021-01-01 12:00:00",+ "2021-01-01 13:00:00",+ "2021-01-01 14:00:00",+ "2021-01-01 15:00:00",+ "2021-01-01 16:00:00",+ "2021-01-01 17:00:00",+ "2021-01-01 18:00:00",+ "2021-01-01 19:00:00",+ "2021-01-01 20:00:00",+ "2021-01-01 21:00:00",+ "2021-01-01 22:00:00",+ "2021-01-01 23:00:00"]end- test "shows last 7 days of visitors", %{conn: conn, site: site} do+ test "shows last 7 days of visitors with time labels", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),build(:pageview, timestamp: ~N[2021-01-07 23:59:00])@@ -1128,13 +1156,53 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => ["2021-01-01", "2021-01-07"],- "dimensions" => ["time"]+ "dimensions" => ["time"],+ "include" => %{"time_labels" => true}})assert json_response(conn, 200)["results"] == [%{"dimensions" => ["2021-01-01"], "metrics" => [1]},%{"dimensions" => ["2021-01-07"], "metrics" => [1]}]++ assert json_response(conn, 200)["meta"]["time_labels"] == [+ "2021-01-01",+ "2021-01-02",+ "2021-01-03",+ "2021-01-04",+ "2021-01-05",+ "2021-01-06",+ "2021-01-07"+ ]+ end++ test "shows weekly data with time labels", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-03 23:59:00]),+ build(:pageview, timestamp: ~N[2021-01-07 23:59:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => ["2020-12-20", "2021-01-07"],+ "dimensions" => ["time:week"],+ "include" => %{"time_labels" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2020-12-28"], "metrics" => [2]},+ %{"dimensions" => ["2021-01-04"], "metrics" => [1]}+ ]++ assert json_response(conn, 200)["meta"]["time_labels"] == [+ "2020-12-20",+ "2020-12-21",+ "2020-12-28",+ "2021-01-04"+ ]endtest "shows last 6 months of visitors", %{conn: conn, site: site} do@@ -1150,7 +1218,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => ["2020-07-01", "2021-01-31"],- "dimensions" => ["time"]+ "dimensions" => ["time:month"]})assert json_response(conn, 200)["results"] == [@@ -1173,7 +1241,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => ["2020-01-01", "2021-01-01"],- "dimensions" => ["time"]+ "dimensions" => ["time:month"]})assert json_response(conn, 200)["results"] == [@@ -1196,7 +1264,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => ["2020-01-01", "2021-01-07"],- "dimensions" => ["time:day"]+ "dimensions" => ["time:day"],+ "include" => %{"time_labels" => true}})assert json_response(conn, 200)["results"] == [@@ -1204,6 +1273,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do%{"dimensions" => ["2020-12-31"], "metrics" => [1]},%{"dimensions" => ["2021-01-01"], "metrics" => [2]}]++ assert length(json_response(conn, 200)["meta"]["time_labels"]) == 373endtest "shows a custom range with daily interval", %{conn: conn, site: site} do@@ -3982,11 +4053,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do})assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["2021-01-01T00:00:00Z", "Google"], "metrics" => [1]},- %{"dimensions" => ["2021-01-02T00:00:00Z", "Google"], "metrics" => [1]},- %{"dimensions" => ["2021-01-02T00:00:00Z", "Direct / None"], "metrics" => [1]},- %{"dimensions" => ["2021-01-03T00:00:00Z", "Direct / None"], "metrics" => [1]},- %{"dimensions" => ["2021-01-03T00:00:00Z", "Twitter"], "metrics" => [1]}+ %{"dimensions" => ["2021-01-01 00:00:00", "Google"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-02 00:00:00", "Google"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-02 00:00:00", "Direct / None"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-03 00:00:00", "Direct / None"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-03 00:00:00", "Twitter"], "metrics" => [1]}]endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exsindex 2a60ea2f74..bf8b39e73e 100644--- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs@@ -131,7 +131,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 02:00:00",@@ -139,7 +139,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 03:00:00",@@ -147,7 +147,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 04:00:00",@@ -155,7 +155,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 05:00:00",@@ -163,7 +163,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 06:00:00",@@ -171,7 +171,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 07:00:00",@@ -179,7 +179,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 08:00:00",@@ -187,7 +187,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 09:00:00",@@ -195,7 +195,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 10:00:00",@@ -203,7 +203,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 11:00:00",@@ -211,7 +211,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 12:00:00",@@ -219,7 +219,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 13:00:00",@@ -227,7 +227,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 14:00:00",@@ -235,7 +235,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 15:00:00",@@ -243,7 +243,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 16:00:00",@@ -251,7 +251,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 17:00:00",@@ -259,7 +259,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 18:00:00",@@ -267,7 +267,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 19:00:00",@@ -275,7 +275,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 20:00:00",@@ -283,7 +283,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 21:00:00",@@ -291,7 +291,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 22:00:00",@@ -299,7 +299,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 0,"pageviews" => 0,"visit_duration" => nil,- "bounce_rate" => nil+ "bounce_rate" => 0},%{"date" => "2021-01-01 23:00:00",@@ -1712,7 +1712,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visits" => 1},%{- "bounce_rate" => nil,+ "bounce_rate" => 0,"date" => "2021-01-02","events" => 0,"pageviews" => 0,diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exsindex d4b0c36e8d..f04cbacbb0 100644--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs@@ -480,13 +480,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert %{"plot" => plot} = json_response(conn, 200)- expected_plot =- if FunWithFlags.enabled?(:experimental_session_count) do- ~w[1 1 1 1 1 2 2 2 2 2 2 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 1 1 0 0]- else- ~w[0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 1 0 0]- end-+ expected_plot = ~w[1 1 1 1 1 2 2 2 2 2 2 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 1 1 0 0]assert plot == Enum.map(expected_plot, &String.to_integer/1)end@@ -565,11 +559,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert %{"plot" => plot} = json_response(conn, 200)- if FunWithFlags.enabled?(:experimental_session_count) do- assert plot == [1, 1, 0, 0, 0]- else- assert plot == [1, 1, 0, 0, 1]- end+ assert plot == [1, 1, 0, 0, 0]endenddiff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exsindex 1f7d5b4868..a873404032 100644--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs@@ -85,7 +85,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest doconn =get(conn,- "/api/stats/#{site.domain}/top-stats?period=day&date=2021-01-01&experimental_session_count=true"+ "/api/stats/#{site.domain}/top-stats?period=day&date=2021-01-01")res = json_response(conn, 200)diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exsindex a2a95ab762..bb4f7687a9 100644--- a/test/plausible_web/controllers/stats_controller_test.exs+++ b/test/plausible_web/controllers/stats_controller_test.exs@@ -323,9 +323,9 @@ defmodule PlausibleWeb.StatsControllerTest do"visit_duration"],["2021-09-20", "1", "1", "1", "1.0", "100", "0"],- ["2021-09-27", "0", "0", "0", "0.0", "", ""],- ["2021-10-04", "0", "0", "0", "0.0", "", ""],- ["2021-10-11", "0", "0", "0", "0.0", "", ""],+ ["2021-09-27", "0", "0", "0", "0.0", "0.0", ""],+ ["2021-10-04", "0", "0", "0", "0.0", "0.0", ""],+ ["2021-10-11", "0", "0", "0", "0.0", "0.0", ""],["2021-10-18", "3", "4", "3", "1.33", "33", "40"],[""]]diff --git a/test/test_helper.exs b/test/test_helper.exsindex d9ec1e60a1..60cb5701a9 100644--- a/test/test_helper.exs+++ b/test/test_helper.exs@@ -11,10 +11,8 @@ Application.ensure_all_started(:double)# Temporary flag to test `experimental_reduced_joins` flag on all tests.if System.get_env("TEST_EXPERIMENTAL_REDUCED_JOINS") == "1" doFunWithFlags.enable(:experimental_reduced_joins)- FunWithFlags.enable(:experimental_session_count)elseFunWithFlags.disable(:experimental_reduced_joins)- FunWithFlags.disable(:experimental_session_count)endEcto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)