Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Fix visitors.csv vs main graph discrepancy (#5230)

RobertJoonasc7bf54325e62
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b468702056..c4123cd2d1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,6 +36,7 @@ All notable changes to this project will be documented in this file.
### Fixed
+- Fix `visitors.csv` (in dashboard CSV export) vs dashboard main graph reporting different results for `visitors` and `visits` with a `time:minute` interval.
- The tracker script now sends pageviews when a page gets loaded from bfcache
- Fix returning filter suggestions for multiple custom property values in the dashboard Filter modal
- Fix typo on login screen
diff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.ts
index e568c0c31a..c410e50cb6 100644
--- a/assets/js/types/query-api.d.ts
+++ b/assets/js/types/query-api.d.ts
@@ -62,7 +62,7 @@ export type SimpleFilterDimensions =
| "visit:exit_page_hostname";
export type CustomPropertyFilterDimensions = string;
export type GoalDimension = "event:goal";
-export type TimeDimensions = "time" | "time:month" | "time:week" | "time:day" | "time:hour";
+export type TimeDimensions = ("time" | "time:month" | "time:week" | "time:day" | "time:hour") | "time:minute";
export type FilterTree = FilterEntry | FilterAndOr | FilterNot | FilterHasDone;
export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern | FilterForSegment;
/**
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex
index ff5411873c..2bebe8658d 100644
--- a/lib/plausible/stats/filters/query_parser.ex
+++ b/lib/plausible/stats/filters/query_parser.ex
@@ -349,6 +349,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
defp parse_time("time"), do: {:ok, "time"}
+ defp parse_time("time:minute"), do: {:ok, "time:minute"}
defp parse_time("time:hour"), do: {:ok, "time:hour"}
defp parse_time("time:day"), do: {:ok, "time:day"}
defp parse_time("time:week"), do: {:ok, "time:week"}
diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex
index 1449447e08..74361d5afd 100644
--- a/lib/plausible/stats/sql/query_builder.ex
+++ b/lib/plausible/stats/sql/query_builder.ex
@@ -182,12 +182,14 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
do: sessions_q |> build_order_by(events_query)
defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) do
- join(subquery(events_q), :left, [e], s in subquery(sessions_q),
+ {join_type, events_q_fields, sessions_q_fields} =
+ TableDecider.join_options(events_query, sessions_query)
+
+ join(subquery(events_q), join_type, [e], s in subquery(sessions_q),
on: ^build_group_by_join(events_query)
)
- |> select_join_fields(events_query, events_query.dimensions, e)
- |> select_join_fields(events_query, events_query.metrics, e)
- |> select_join_fields(sessions_query, List.delete(sessions_query.metrics, :sample_percent), s)
+ |> select_join_fields(events_query, events_q_fields, e)
+ |> select_join_fields(sessions_query, sessions_q_fields, s)
|> build_order_by(events_query)
end
diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.ex
index 7a936630f1..2873493c1b 100644
--- a/lib/plausible/stats/table_decider.ex
+++ b/lib/plausible/stats/table_decider.ex
@@ -54,6 +54,39 @@ defmodule Plausible.Stats.TableDecider do
end
end
+ @doc """
+ Returns a three-element tuple with instructions on how to join two Ecto
+ queries. The arguments (`events_query` and `sessions_query`) are `%Query{}`
+ structs that have been split by TableDecider already.
+
+ Normally we can always LEFT JOIN sessions to events, selecting `dimensions`
+ only from the events subquery. That's because:
+
+ 1) session dimensions (e.g. entry_page) cannot be queried alongside event
+ metrics/dimensions, or
+
+ 2) session dimensions (e.g. operating_system) are also available in the
+ events table.
+
+ The only exception is using the "time:minute" dimension where the sessions
+ subquery might return more rows than the events one. That's because we're
+ counting sessions in all time buckets they were active in.
+ """
+ def join_options(events_query, sessions_query) do
+ events_q_select_fields = events_query.metrics ++ events_query.dimensions
+ sessions_q_select_fields = sessions_query.metrics -- [:sample_percent]
+
+ if "time:minute" in events_query.dimensions do
+ {
+ :full,
+ events_q_select_fields -- ["time:minute"],
+ sessions_q_select_fields ++ ["time:minute"]
+ }
+ else
+ {:left, events_q_select_fields, sessions_q_select_fields}
+ end
+ end
+
def partition_metrics(metrics, query) do
%{
event: event_only_metrics,
@@ -99,10 +132,12 @@ defmodule Plausible.Stats.TableDecider do
defp metric_partitioner(%Query{legacy_breakdown: true}, :pageviews), do: :either
defp metric_partitioner(%Query{legacy_breakdown: true}, :events), do: :either
+ defp metric_partitioner(query, metric) when metric in [:visitors, :visits] do
+ if "time:minute" in query.dimensions, do: :session, else: :either
+ end
+
defp metric_partitioner(_, :conversion_rate), do: :either
defp metric_partitioner(_, :group_conversion_rate), do: :either
- defp metric_partitioner(_, :visitors), do: :either
- defp metric_partitioner(_, :visits), do: :either
defp metric_partitioner(_, :percentage), do: :either
defp metric_partitioner(_, :average_revenue), do: :event
diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json
index 5a24d82afb..6c7571bcbb 100644
--- a/priv/json-schemas/query-api-schema.json
+++ b/priv/json-schemas/query-api-schema.json
@@ -324,8 +324,16 @@
"markdownDescription": "Goal dimension"
},
"time_dimensions": {
- "type": "string",
- "enum": ["time", "time:month", "time:week", "time:day", "time:hour"]
+ "oneOf": [
+ {
+ "type": "string",
+ "enum": ["time", "time:month", "time:week", "time:day", "time:hour"]
+ },
+ {
+ "const": "time:minute",
+ "$comment": "only :internal"
+ }
+ ]
},
"dimensions": {
"oneOf": [
diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs
index 14b3297fa3..9cfbb0bf28 100644
--- a/test/plausible/stats/query_parser_test.exs
+++ b/test/plausible/stats/query_parser_test.exs
@@ -1490,6 +1490,39 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
end
end
+ test "time:minute dimension fails public schema validation", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["time:minute"]
+ }
+ |> check_error(site, "#/dimensions/0: Invalid dimension \"time:minute\"")
+ end
+
+ test "time:minute dimension passes internal schema validation", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["time:minute"]
+ }
+ |> check_success(
+ site,
+ %{
+ metrics: [:visitors],
+ utc_time_range: @date_range_day,
+ filters: [],
+ dimensions: ["time:minute"],
+ order_by: nil,
+ timezone: site.timezone,
+ include: @default_include,
+ pagination: %{limit: 10_000, offset: 0}
+ },
+ :internal
+ )
+ end
+
test "custom properties dimension", %{site: site} do
%{
"site_id" => site.domain,
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
index 921289a2a8..083120a0d9 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
@@ -1342,6 +1342,36 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
end
describe "timeseries" do
+ test "breakdown by time:minute (internal API), counts visitors and visits in all buckets their session was active in",
+ %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:10:00])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "visits", "pageviews"],
+ "date_range" => ["2021-01-01T00:00:00Z", "2021-01-01T00:10:00Z"],
+ "dimensions" => ["time:minute"]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["2021-01-01 00:00:00"], "metrics" => [1, 1, 1]},
+ %{"dimensions" => ["2021-01-01 00:01:00"], "metrics" => [1, 1, 0]},
+ %{"dimensions" => ["2021-01-01 00:02:00"], "metrics" => [1, 1, 0]},
+ %{"dimensions" => ["2021-01-01 00:03:00"], "metrics" => [1, 1, 0]},
+ %{"dimensions" => ["2021-01-01 00:04:00"], "metrics" => [1, 1, 0]},
+ %{"dimensions" => ["2021-01-01 00:05:00"], "metrics" => [1, 1, 0]},
+ %{"dimensions" => ["2021-01-01 00:06:00"], "metrics" => [1, 1, 0]},
+ %{"dimensions" => ["2021-01-01 00:07:00"], "metrics" => [1, 1, 0]},
+ %{"dimensions" => ["2021-01-01 00:08:00"], "metrics" => [1, 1, 0]},
+ %{"dimensions" => ["2021-01-01 00:09:00"], "metrics" => [1, 1, 0]},
+ %{"dimensions" => ["2021-01-01 00:10:00"], "metrics" => [1, 1, 1]}
+ ]
+ end
+
test "shows hourly data for a certain date with time_labels", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),