Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Fix: time series trimming (#5705)
Karl-Aksel Puulmann60a47f336292
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex d04ba3412a..a7411880a1 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -27,12 +27,8 @@ defmodule Plausible.Stats.Filters.QueryParser dodef default_include(), do: @default_includedef parse(site, schema_type, params, now \\ nil) when is_map(params) do- {now, date} =- if now do- {now, DateTime.shift_zone!(now, site.timezone) |> DateTime.to_date()}- else- {DateTime.utc_now(:second), today(site)}- end+ now = now || Plausible.Stats.Query.Test.get_fixed_now()+ date = now |> DateTime.shift_zone!(site.timezone) |> DateTime.to_date()with :ok <- JSONSchema.validate(schema_type, params),{:ok, date, now} <- parse_date(site, Map.get(params, "date"), date, now),@@ -289,8 +285,6 @@ defmodule Plausible.Stats.Filters.QueryParser doendend- defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date()-defp parse_dimensions(dimensions) when is_list(dimensions) doparse_list(dimensions,diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.exindex 72b33082f9..b0140b99e1 100644--- a/lib/plausible/stats/legacy/legacy_query_builder.ex+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex@@ -11,7 +11,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder doalias Plausible.Stats.{Filters, Interval, Query, DateTimeRange}def from(site, params, debug_metadata, now \\ nil) do- now = now || DateTime.utc_now(:second)+ now = now || Plausible.Stats.Query.Test.get_fixed_now()query =Querydiff --git a/lib/plausible/stats/query/test.ex b/lib/plausible/stats/query/test.exnew file mode 100644index 0000000000..84e6e1fd7b--- /dev/null+++ b/lib/plausible/stats/query/test.ex@@ -0,0 +1,15 @@+defmodule Plausible.Stats.Query.Test do+ @moduledoc """+ Module used in tests to 'set' the current time.+ """++ @now_key :__now++ def fix_now(now) do+ Process.put(@now_key, now)+ end++ def get_fixed_now() do+ Process.get(@now_key) || DateTime.utc_now(:second)+ end+enddiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 380d448f50..9ea7e19a0f 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -7,7 +7,7 @@ defmodule Plausible.Stats.Timeseries douse Plausibleuse Plausible.ClickhouseRepo- alias Plausible.Stats.{Comparisons, Query, QueryRunner, Metrics, Time}+ alias Plausible.Stats.{Comparisons, Query, QueryRunner, Metrics, Time, QueryOptimizer}@time_dimension %{"month" => "time:month",@@ -26,6 +26,7 @@ defmodule Plausible.Stats.Timeseries doorder_by: [{time_dimension(query), :asc}],remove_unavailable_revenue_metrics: true)+ |> QueryOptimizer.optimize()comparison_query =if(query.include.comparisons,diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exsindex ed584fcf9b..3720789b61 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs@@ -185,12 +185,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryComparisonsTest dobuild(:pageview, timestamp: ~N[2022-07-01 00:00:00])])+ Plausible.Stats.Query.Test.fix_now(~U[2022-07-01 14:00:00Z])+conn =post(conn, "/api/v2/query-internal-test", %{"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "91d",- "date" => "2022-07-01","dimensions" => ["time:day"],"include" => %{"time_labels" => true,diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exsindex 0c9ebc1398..e703430afb 100644--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs@@ -1340,6 +1340,95 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert Enum.at(plot, Enum.find_index(labels, &(&1 == "2023-03-01 12:00:00"))) == 1end++ test "trims hourly relative date range", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-08 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-08 06:05:00]),+ build(:pageview, timestamp: ~N[2021-01-08 08:59:00]),+ build(:pageview, timestamp: ~N[2021-01-08 23:59:00])+ ])++ Plausible.Stats.Query.Test.fix_now(~U[2021-01-08 08:05:00Z])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=day&metric=visitors&date=2021-01-08&interval=hour"+ )++ assert_matches %{+ "labels" => [+ "2021-01-08 00:00:00",+ "2021-01-08 01:00:00",+ "2021-01-08 02:00:00",+ "2021-01-08 03:00:00",+ "2021-01-08 04:00:00",+ "2021-01-08 05:00:00",+ "2021-01-08 06:00:00",+ "2021-01-08 07:00:00",+ "2021-01-08 08:00:00"+ ],+ "plot" => [1, 0, 0, 0, 0, 0, 1, 0, 1]+ } = json_response(conn, 200)+ end++ test "trims monthly relative date range", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-05 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00])+ ])++ Plausible.Stats.Query.Test.fix_now(~U[2021-01-07 12:00:00Z])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=month&metric=visitors&date=2021-01-07&interval=day"+ )++ assert_matches %{+ "labels" => [+ "2021-01-01",+ "2021-01-02",+ "2021-01-03",+ "2021-01-04",+ "2021-01-05",+ "2021-01-06",+ "2021-01-07"+ ],+ "plot" => [1, 0, 0, 0, 1, 0, 1]+ } = json_response(conn, 200)+ end++ test "trims yearly relative date range", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-05 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-30 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00]),+ build(:pageview, timestamp: ~N[2021-02-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-02-09 00:00:00])+ ])++ Plausible.Stats.Query.Test.fix_now(~U[2021-02-07 12:00:00Z])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=year&metric=visitors&date=2021-02-07&interval=month"+ )++ assert_matches %{+ "labels" => [+ "2021-01-01",+ "2021-02-01"+ ],+ "plot" => [4, 1]+ } = json_response(conn, 200)+ endenddescribe "GET /api/stats/main-graph - comparisons" do@@ -1513,6 +1602,104 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert this_week_plot == [50.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]assert last_week_plot == [33.33, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]end++ test "does not trim hourly relative date range when comparing", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-08 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-08 06:05:00]),+ build(:pageview, timestamp: ~N[2021-01-08 08:59:00]),+ build(:pageview, timestamp: ~N[2021-01-08 23:59:00])+ ])++ Plausible.Stats.Query.Test.fix_now(~U[2021-01-08 08:05:00Z])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=day&metric=visitors&date=2021-01-08&interval=hour&comparison=previous_period"+ )++ assert_matches %{+ "labels" => [+ "2021-01-08 00:00:00",+ "2021-01-08 01:00:00",+ "2021-01-08 02:00:00",+ "2021-01-08 03:00:00",+ "2021-01-08 04:00:00",+ "2021-01-08 05:00:00",+ "2021-01-08 06:00:00",+ "2021-01-08 07:00:00",+ "2021-01-08 08:00:00",+ "2021-01-08 09:00:00",+ "2021-01-08 10:00:00",+ "2021-01-08 11:00:00",+ "2021-01-08 12:00:00",+ "2021-01-08 13:00:00",+ "2021-01-08 14:00:00",+ "2021-01-08 15:00:00",+ "2021-01-08 16:00:00",+ "2021-01-08 17:00:00",+ "2021-01-08 18:00:00",+ "2021-01-08 19:00:00",+ "2021-01-08 20:00:00",+ "2021-01-08 21:00:00",+ "2021-01-08 22:00:00",+ "2021-01-08 23:00:00"+ ],+ "plot" => [+ 1,+ 0,+ 0,+ 0,+ 0,+ 0,+ 1,+ 0,+ 1,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 1+ ],+ "comparison_plot" => [+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0,+ 0+ ]+ } = json_response(conn, 200)+ endenddescribe "GET /api/stats/main-graph - total_revenue plot" do