Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Fix visitors.csv vs main graph discrepancy (#5230)
RobertJoonasc7bf54325e62
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex b468702056..c4123cd2d1 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -36,6 +36,7 @@ All notable changes to this project will be documented in this file.### Fixed+- Fix `visitors.csv` (in dashboard CSV export) vs dashboard main graph reporting different results for `visitors` and `visits` with a `time:minute` interval.- The tracker script now sends pageviews when a page gets loaded from bfcache- Fix returning filter suggestions for multiple custom property values in the dashboard Filter modal- Fix typo on login screendiff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.tsindex e568c0c31a..c410e50cb6 100644--- a/assets/js/types/query-api.d.ts+++ b/assets/js/types/query-api.d.ts@@ -62,7 +62,7 @@ export type SimpleFilterDimensions =| "visit:exit_page_hostname";export type CustomPropertyFilterDimensions = string;export type GoalDimension = "event:goal";-export type TimeDimensions = "time" | "time:month" | "time:week" | "time:day" | "time:hour";+export type TimeDimensions = ("time" | "time:month" | "time:week" | "time:day" | "time:hour") | "time:minute";export type FilterTree = FilterEntry | FilterAndOr | FilterNot | FilterHasDone;export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern | FilterForSegment;/**diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex ff5411873c..2bebe8658d 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -349,6 +349,7 @@ defmodule Plausible.Stats.Filters.QueryParser doenddefp parse_time("time"), do: {:ok, "time"}+ defp parse_time("time:minute"), do: {:ok, "time:minute"}defp parse_time("time:hour"), do: {:ok, "time:hour"}defp parse_time("time:day"), do: {:ok, "time:day"}defp parse_time("time:week"), do: {:ok, "time:week"}diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex 1449447e08..74361d5afd 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -182,12 +182,14 @@ defmodule Plausible.Stats.SQL.QueryBuilder dodo: sessions_q |> build_order_by(events_query)defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) do- join(subquery(events_q), :left, [e], s in subquery(sessions_q),+ {join_type, events_q_fields, sessions_q_fields} =+ TableDecider.join_options(events_query, sessions_query)++ join(subquery(events_q), join_type, [e], s in subquery(sessions_q),on: ^build_group_by_join(events_query))- |> select_join_fields(events_query, events_query.dimensions, e)- |> select_join_fields(events_query, events_query.metrics, e)- |> select_join_fields(sessions_query, List.delete(sessions_query.metrics, :sample_percent), s)+ |> select_join_fields(events_query, events_q_fields, e)+ |> select_join_fields(sessions_query, sessions_q_fields, s)|> build_order_by(events_query)enddiff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.exindex 7a936630f1..2873493c1b 100644--- a/lib/plausible/stats/table_decider.ex+++ b/lib/plausible/stats/table_decider.ex@@ -54,6 +54,39 @@ defmodule Plausible.Stats.TableDecider doendend+ @doc """+ Returns a three-element tuple with instructions on how to join two Ecto+ queries. The arguments (`events_query` and `sessions_query`) are `%Query{}`+ structs that have been split by TableDecider already.++ Normally we can always LEFT JOIN sessions to events, selecting `dimensions`+ only from the events subquery. That's because:++ 1) session dimensions (e.g. entry_page) cannot be queried alongside event+ metrics/dimensions, or++ 2) session dimensions (e.g. operating_system) are also available in the+ events table.++ The only exception is using the "time:minute" dimension where the sessions+ subquery might return more rows than the events one. That's because we're+ counting sessions in all time buckets they were active in.+ """+ def join_options(events_query, sessions_query) do+ events_q_select_fields = events_query.metrics ++ events_query.dimensions+ sessions_q_select_fields = sessions_query.metrics -- [:sample_percent]++ if "time:minute" in events_query.dimensions do+ {+ :full,+ events_q_select_fields -- ["time:minute"],+ sessions_q_select_fields ++ ["time:minute"]+ }+ else+ {:left, events_q_select_fields, sessions_q_select_fields}+ end+ end+def partition_metrics(metrics, query) do%{event: event_only_metrics,@@ -99,10 +132,12 @@ defmodule Plausible.Stats.TableDecider dodefp metric_partitioner(%Query{legacy_breakdown: true}, :pageviews), do: :eitherdefp metric_partitioner(%Query{legacy_breakdown: true}, :events), do: :either+ defp metric_partitioner(query, metric) when metric in [:visitors, :visits] do+ if "time:minute" in query.dimensions, do: :session, else: :either+ end+defp metric_partitioner(_, :conversion_rate), do: :eitherdefp metric_partitioner(_, :group_conversion_rate), do: :either- defp metric_partitioner(_, :visitors), do: :either- defp metric_partitioner(_, :visits), do: :eitherdefp metric_partitioner(_, :percentage), do: :eitherdefp metric_partitioner(_, :average_revenue), do: :eventdiff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.jsonindex 5a24d82afb..6c7571bcbb 100644--- a/priv/json-schemas/query-api-schema.json+++ b/priv/json-schemas/query-api-schema.json@@ -324,8 +324,16 @@"markdownDescription": "Goal dimension"},"time_dimensions": {- "type": "string",- "enum": ["time", "time:month", "time:week", "time:day", "time:hour"]+ "oneOf": [+ {+ "type": "string",+ "enum": ["time", "time:month", "time:week", "time:day", "time:hour"]+ },+ {+ "const": "time:minute",+ "$comment": "only :internal"+ }+ ]},"dimensions": {"oneOf": [diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex 14b3297fa3..9cfbb0bf28 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -1490,6 +1490,39 @@ defmodule Plausible.Stats.Filters.QueryParserTest doendend+ test "time:minute dimension fails public schema validation", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["time:minute"]+ }+ |> check_error(site, "#/dimensions/0: Invalid dimension \"time:minute\"")+ end++ test "time:minute dimension passes internal schema validation", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["time:minute"]+ }+ |> check_success(+ site,+ %{+ metrics: [:visitors],+ utc_time_range: @date_range_day,+ filters: [],+ dimensions: ["time:minute"],+ order_by: nil,+ timezone: site.timezone,+ include: @default_include,+ pagination: %{limit: 10_000, offset: 0}+ },+ :internal+ )+ end+test "custom properties dimension", %{site: site} do%{"site_id" => site.domain,diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex 921289a2a8..083120a0d9 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -1342,6 +1342,36 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doenddescribe "timeseries" do+ test "breakdown by time:minute (internal API), counts visitors and visits in all buckets their session was active in",+ %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:10:00])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "visits", "pageviews"],+ "date_range" => ["2021-01-01T00:00:00Z", "2021-01-01T00:10:00Z"],+ "dimensions" => ["time:minute"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2021-01-01 00:00:00"], "metrics" => [1, 1, 1]},+ %{"dimensions" => ["2021-01-01 00:01:00"], "metrics" => [1, 1, 0]},+ %{"dimensions" => ["2021-01-01 00:02:00"], "metrics" => [1, 1, 0]},+ %{"dimensions" => ["2021-01-01 00:03:00"], "metrics" => [1, 1, 0]},+ %{"dimensions" => ["2021-01-01 00:04:00"], "metrics" => [1, 1, 0]},+ %{"dimensions" => ["2021-01-01 00:05:00"], "metrics" => [1, 1, 0]},+ %{"dimensions" => ["2021-01-01 00:06:00"], "metrics" => [1, 1, 0]},+ %{"dimensions" => ["2021-01-01 00:07:00"], "metrics" => [1, 1, 0]},+ %{"dimensions" => ["2021-01-01 00:08:00"], "metrics" => [1, 1, 0]},+ %{"dimensions" => ["2021-01-01 00:09:00"], "metrics" => [1, 1, 0]},+ %{"dimensions" => ["2021-01-01 00:10:00"], "metrics" => [1, 1, 1]}+ ]+ end+test "shows hourly data for a certain date with time_labels", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),