Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Fix: time series trimming (#5705)

Karl-Aksel Puulmann60a47f336292
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex
index d04ba3412a..a7411880a1 100644
--- a/lib/plausible/stats/filters/query_parser.ex
+++ b/lib/plausible/stats/filters/query_parser.ex
@@ -27,12 +27,8 @@ defmodule Plausible.Stats.Filters.QueryParser do
def default_include(), do: @default_include
def parse(site, schema_type, params, now \\ nil) when is_map(params) do
- {now, date} =
- if now do
- {now, DateTime.shift_zone!(now, site.timezone) |> DateTime.to_date()}
- else
- {DateTime.utc_now(:second), today(site)}
- end
+ now = now || Plausible.Stats.Query.Test.get_fixed_now()
+ date = now |> DateTime.shift_zone!(site.timezone) |> DateTime.to_date()
with :ok <- JSONSchema.validate(schema_type, params),
{:ok, date, now} <- parse_date(site, Map.get(params, "date"), date, now),
@@ -289,8 +285,6 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
end
- defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date()
-
defp parse_dimensions(dimensions) when is_list(dimensions) do
parse_list(
dimensions,
diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.ex
index 72b33082f9..b0140b99e1 100644
--- a/lib/plausible/stats/legacy/legacy_query_builder.ex
+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex
@@ -11,7 +11,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
alias Plausible.Stats.{Filters, Interval, Query, DateTimeRange}
def from(site, params, debug_metadata, now \\ nil) do
- now = now || DateTime.utc_now(:second)
+ now = now || Plausible.Stats.Query.Test.get_fixed_now()
query =
Query
diff --git a/lib/plausible/stats/query/test.ex b/lib/plausible/stats/query/test.ex
new file mode 100644
index 0000000000..84e6e1fd7b
--- /dev/null
+++ b/lib/plausible/stats/query/test.ex
@@ -0,0 +1,15 @@
+defmodule Plausible.Stats.Query.Test do
+ @moduledoc """
+ Module used in tests to 'set' the current time.
+ """
+
+ @now_key :__now
+
+ def fix_now(now) do
+ Process.put(@now_key, now)
+ end
+
+ def get_fixed_now() do
+ Process.get(@now_key) || DateTime.utc_now(:second)
+ end
+end
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index 380d448f50..9ea7e19a0f 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -7,7 +7,7 @@ defmodule Plausible.Stats.Timeseries do
use Plausible
use Plausible.ClickhouseRepo
- alias Plausible.Stats.{Comparisons, Query, QueryRunner, Metrics, Time}
+ alias Plausible.Stats.{Comparisons, Query, QueryRunner, Metrics, Time, QueryOptimizer}
@time_dimension %{
"month" => "time:month",
@@ -26,6 +26,7 @@ defmodule Plausible.Stats.Timeseries do
order_by: [{time_dimension(query), :asc}],
remove_unavailable_revenue_metrics: true
)
+ |> QueryOptimizer.optimize()
comparison_query =
if(query.include.comparisons,
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs
index ed584fcf9b..3720789b61 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs
@@ -185,12 +185,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryComparisonsTest do
build(:pageview, timestamp: ~N[2022-07-01 00:00:00])
])
+ Plausible.Stats.Query.Test.fix_now(~U[2022-07-01 14:00:00Z])
+
conn =
post(conn, "/api/v2/query-internal-test", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "91d",
- "date" => "2022-07-01",
"dimensions" => ["time:day"],
"include" => %{
"time_labels" => true,
diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
index 0c9ebc1398..e703430afb 100644
--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
@@ -1340,6 +1340,95 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert Enum.at(plot, Enum.find_index(labels, &(&1 == "2023-03-01 12:00:00"))) == 1
end
+
+ test "trims hourly relative date range", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-08 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-08 06:05:00]),
+ build(:pageview, timestamp: ~N[2021-01-08 08:59:00]),
+ build(:pageview, timestamp: ~N[2021-01-08 23:59:00])
+ ])
+
+ Plausible.Stats.Query.Test.fix_now(~U[2021-01-08 08:05:00Z])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=day&metric=visitors&date=2021-01-08&interval=hour"
+ )
+
+ assert_matches %{
+ "labels" => [
+ "2021-01-08 00:00:00",
+ "2021-01-08 01:00:00",
+ "2021-01-08 02:00:00",
+ "2021-01-08 03:00:00",
+ "2021-01-08 04:00:00",
+ "2021-01-08 05:00:00",
+ "2021-01-08 06:00:00",
+ "2021-01-08 07:00:00",
+ "2021-01-08 08:00:00"
+ ],
+ "plot" => [1, 0, 0, 0, 0, 0, 1, 0, 1]
+ } = json_response(conn, 200)
+ end
+
+ test "trims monthly relative date range", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-05 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-07 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00])
+ ])
+
+ Plausible.Stats.Query.Test.fix_now(~U[2021-01-07 12:00:00Z])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=month&metric=visitors&date=2021-01-07&interval=day"
+ )
+
+ assert_matches %{
+ "labels" => [
+ "2021-01-01",
+ "2021-01-02",
+ "2021-01-03",
+ "2021-01-04",
+ "2021-01-05",
+ "2021-01-06",
+ "2021-01-07"
+ ],
+ "plot" => [1, 0, 0, 0, 1, 0, 1]
+ } = json_response(conn, 200)
+ end
+
+ test "trims yearly relative date range", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-05 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-30 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-02-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-02-09 00:00:00])
+ ])
+
+ Plausible.Stats.Query.Test.fix_now(~U[2021-02-07 12:00:00Z])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=year&metric=visitors&date=2021-02-07&interval=month"
+ )
+
+ assert_matches %{
+ "labels" => [
+ "2021-01-01",
+ "2021-02-01"
+ ],
+ "plot" => [4, 1]
+ } = json_response(conn, 200)
+ end
end
describe "GET /api/stats/main-graph - comparisons" do
@@ -1513,6 +1602,104 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert this_week_plot == [50.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
assert last_week_plot == [33.33, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
end
+
+ test "does not trim hourly relative date range when comparing", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-08 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-08 06:05:00]),
+ build(:pageview, timestamp: ~N[2021-01-08 08:59:00]),
+ build(:pageview, timestamp: ~N[2021-01-08 23:59:00])
+ ])
+
+ Plausible.Stats.Query.Test.fix_now(~U[2021-01-08 08:05:00Z])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=day&metric=visitors&date=2021-01-08&interval=hour&comparison=previous_period"
+ )
+
+ assert_matches %{
+ "labels" => [
+ "2021-01-08 00:00:00",
+ "2021-01-08 01:00:00",
+ "2021-01-08 02:00:00",
+ "2021-01-08 03:00:00",
+ "2021-01-08 04:00:00",
+ "2021-01-08 05:00:00",
+ "2021-01-08 06:00:00",
+ "2021-01-08 07:00:00",
+ "2021-01-08 08:00:00",
+ "2021-01-08 09:00:00",
+ "2021-01-08 10:00:00",
+ "2021-01-08 11:00:00",
+ "2021-01-08 12:00:00",
+ "2021-01-08 13:00:00",
+ "2021-01-08 14:00:00",
+ "2021-01-08 15:00:00",
+ "2021-01-08 16:00:00",
+ "2021-01-08 17:00:00",
+ "2021-01-08 18:00:00",
+ "2021-01-08 19:00:00",
+ "2021-01-08 20:00:00",
+ "2021-01-08 21:00:00",
+ "2021-01-08 22:00:00",
+ "2021-01-08 23:00:00"
+ ],
+ "plot" => [
+ 1,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 1,
+ 0,
+ 1,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 1
+ ],
+ "comparison_plot" => [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ]
+ } = json_response(conn, 200)
+ end
end
describe "GET /api/stats/main-graph - total_revenue plot" do