Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Make subsequent query boundaries idempotent (#3723)
hq10fa7f3c1e10d
diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex e9208088e3..aad991b2c4 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -484,28 +484,28 @@ defmodule Plausible.Stats.Base doendend- def utc_boundaries(%Query{period: "realtime"}, site) do+ def utc_boundaries(%Query{period: "realtime", now: now}, site) dolast_datetime =- NaiveDateTime.utc_now()+ now|> Timex.shift(seconds: 5)|> beginning_of_time(site.native_stats_start_at)|> NaiveDateTime.truncate(:second)first_datetime =- NaiveDateTime.utc_now() |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second)+ now |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second){first_datetime, last_datetime}end- def utc_boundaries(%Query{period: "30m"}, site) do+ def utc_boundaries(%Query{period: "30m", now: now}, site) dolast_datetime =- NaiveDateTime.utc_now()+ now|> Timex.shift(seconds: 5)|> beginning_of_time(site.native_stats_start_at)|> NaiveDateTime.truncate(:second)first_datetime =- NaiveDateTime.utc_now() |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second)+ now |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second){first_datetime, last_datetime}enddiff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex ce116f491b..4c0d34c246 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -555,8 +555,8 @@ defmodule Plausible.Stats.Clickhouse dobase_query_bare(site, query) |> include_goal_conversions(query)end- defp utc_boundaries(%Query{period: "30m"}, site) do- last_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)+ defp utc_boundaries(%Query{now: now, period: "30m"}, site) do+ last_datetime = now |> NaiveDateTime.truncate(:second)first_datetime =last_datetime@@ -567,8 +567,8 @@ defmodule Plausible.Stats.Clickhouse do{first_datetime, last_datetime}end- defp utc_boundaries(%Query{period: "realtime"}, site) do- last_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)+ defp utc_boundaries(%Query{now: now, period: "realtime"}, site) do+ last_datetime = now |> NaiveDateTime.truncate(:second)first_datetime =last_datetimediff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex a554201727..d1c7e6575b 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -7,7 +7,8 @@ defmodule Plausible.Stats.Query dofilters: %{},sample_threshold: 20_000_000,imported_data_requested: false,- include_imported: false+ include_imported: false,+ now: nilrequire OpenTelemetry.Tracer, as: Traceralias Plausible.Stats.{FilterParser, Interval}@@ -15,8 +16,12 @@ defmodule Plausible.Stats.Query do@type t :: %__MODULE__{}def from(site, params) do+ now = NaiveDateTime.utc_now(:second)+query =- query_by_period(site, params)+ __MODULE__+ |> struct!(now: now)+ |> put_period(site, params)|> put_interval(params)|> put_parsed_filters(params)|> put_imported_opts(site, params)@@ -29,57 +34,49 @@ defmodule Plausible.Stats.Query doqueryend- defp query_by_period(site, %{"period" => "realtime"}) do+ defp put_period(query, site, %{"period" => "realtime"}) dodate = today(site.timezone)- %__MODULE__{- period: "realtime",- date_range: Date.range(date, date)- }+ struct!(query, period: "realtime", date_range: Date.range(date, date))end- defp query_by_period(site, %{"period" => "day"} = params) do+ defp put_period(query, site, %{"period" => "day"} = params) dodate = parse_single_date(site.timezone, params)- %__MODULE__{- period: "day",- date_range: Date.range(date, date)- }+ struct!(query, period: "day", date_range: Date.range(date, date))end- defp query_by_period(site, %{"period" => "7d"} = params) do+ defp put_period(query, site, %{"period" => "7d"} = params) doend_date = parse_single_date(site.timezone, params)start_date = end_date |> Timex.shift(days: -6)- %__MODULE__{+ struct!(+ query,period: "7d",date_range: Date.range(start_date, end_date)- }+ )end- defp query_by_period(site, %{"period" => "30d"} = params) do+ defp put_period(query, site, %{"period" => "30d"} = params) doend_date = parse_single_date(site.timezone, params)start_date = end_date |> Timex.shift(days: -30)- %__MODULE__{- period: "30d",- date_range: Date.range(start_date, end_date)- }+ struct!(query, period: "30d", date_range: Date.range(start_date, end_date))end- defp query_by_period(site, %{"period" => "month"} = params) do+ defp put_period(query, site, %{"period" => "month"} = params) dodate = parse_single_date(site.timezone, params)start_date = Timex.beginning_of_month(date)end_date = Timex.end_of_month(date)- %__MODULE__{+ struct!(query,period: "month",date_range: Date.range(start_date, end_date)- }+ )end- defp query_by_period(site, %{"period" => "6mo"} = params) do+ defp put_period(query, site, %{"period" => "6mo"} = params) doend_date =parse_single_date(site.timezone, params)|> Timex.end_of_month()@@ -88,13 +85,13 @@ defmodule Plausible.Stats.Query doTimex.shift(end_date, months: -5)|> Timex.beginning_of_month()- %__MODULE__{+ struct!(query,period: "6mo",date_range: Date.range(start_date, end_date)- }+ )end- defp query_by_period(site, %{"period" => "12mo"} = params) do+ defp put_period(query, site, %{"period" => "12mo"} = params) doend_date =parse_single_date(site.timezone, params)|> Timex.end_of_month()@@ -103,78 +100,77 @@ defmodule Plausible.Stats.Query doTimex.shift(end_date, months: -11)|> Timex.beginning_of_month()- %__MODULE__{+ struct!(query,period: "12mo",date_range: Date.range(start_date, end_date)- }+ )end- defp query_by_period(site, %{"period" => "year"} = params) do+ defp put_period(query, site, %{"period" => "year"} = params) doend_date =parse_single_date(site.timezone, params)|> Timex.end_of_year()start_date = Timex.beginning_of_year(end_date)- %__MODULE__{+ struct!(query,period: "year",date_range: Date.range(start_date, end_date)- }+ )end- defp query_by_period(site, %{"period" => "all"}) do+ defp put_period(query, site, %{"period" => "all"}) donow = today(site.timezone)start_date = Plausible.Site.local_start_date(site) || now- %__MODULE__{+ struct!(query,period: "all",date_range: Date.range(start_date, now)- }+ )end- defp query_by_period(site, %{"period" => "custom", "from" => from, "to" => to} = params) do+ defp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) donew_params =params|> Map.drop(["from", "to"])|> Map.put("date", Enum.join([from, to], ","))- query_by_period(site, new_params)+ put_period(query, site, new_params)end- defp query_by_period(_site, %{"period" => "custom", "date" => date}) do+ defp put_period(query, _site, %{"period" => "custom", "date" => date}) do[from, to] = String.split(date, ",")from_date = Date.from_iso8601!(String.trim(from))to_date = Date.from_iso8601!(String.trim(to))- %__MODULE__{+ struct!(query,period: "custom",date_range: Date.range(from_date, to_date)- }+ )end- defp query_by_period(site, params) do- query_by_period(site, Map.merge(params, %{"period" => "30d"}))+ defp put_period(query, site, params) do+ put_period(query, site, Map.merge(params, %{"period" => "30d"}))enddefp put_interval(%{:period => "all"} = query, params) dointerval = Map.get(params, "interval", Interval.default_for_date_range(query.date_range))- Map.put(query, :interval, interval)+ struct!(query, interval: interval)enddefp put_interval(query, params) dointerval = Map.get(params, "interval", Interval.default_for_period(query.period))- Map.put(query, :interval, interval)+ struct!(query, interval: interval)enddefp put_parsed_filters(query, params) do- Map.put(query, :filters, FilterParser.parse_filters(params["filters"]))+ struct!(query, filters: FilterParser.parse_filters(params["filters"]))enddef put_filter(query, key, val) do- %__MODULE__{- query- | filters: Map.put(query.filters, key, val)- }+ struct!(query,+ filters: Map.put(query.filters, key, val)+ )enddef remove_event_filters(query, opts) do@@ -189,7 +185,7 @@ defmodule Plausible.Stats.Query doend)|> Enum.into(%{})- %__MODULE__{query | filters: new_filters}+ struct!(query, filters: new_filters)enddef has_event_filters?(query) do@@ -220,9 +216,10 @@ defmodule Plausible.Stats.Query dodefp put_imported_opts(query, site, params) dorequested? = params["with_imported"] == "true"- query- |> Map.put(:imported_data_requested, requested?)- |> Map.put(:include_imported, include_imported?(query, site, requested?))+ struct!(query,+ imported_data_requested: requested?,+ include_imported: include_imported?(query, site, requested?)+ )enddefp maybe_drop_prop_filter(query, site) do@@ -234,7 +231,7 @@ defmodule Plausible.Stats.Query doendif prop_filter? && !props_available?.(),- do: %__MODULE__{query | filters: Map.drop(query.filters, ["props"])},+ do: struct!(query, filters: Map.drop(query.filters, ["props"])),else: queryenddiff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exsindex 49b4033c26..b609f8875f 100644--- a/test/plausible/stats/query_test.exs+++ b/test/plausible/stats/query_test.exs@@ -15,6 +15,19 @@ defmodule Plausible.Stats.QueryTest do{:ok, site: site, user: user}end+ @tag :slow+ test "keeps current timestamp so that utc_boundaries don't depend on time passing by", %{+ site: site+ } do+ q1 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "realtime"})+ q2 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "30m"})+ boundaries1 = Plausible.Stats.Base.utc_boundaries(q1, site)+ boundaries2 = Plausible.Stats.Base.utc_boundaries(q2, site)+ :timer.sleep(1500)+ assert ^boundaries1 = Plausible.Stats.Base.utc_boundaries(q1, site)+ assert ^boundaries2 = Plausible.Stats.Base.utc_boundaries(q2, site)+ end+test "parses day format", %{site: site} doq = Query.from(site, %{"period" => "day", "date" => "2019-01-01"})