Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
APIv2: Implement pagination and `include.total_rows` (#4575)
Karl-Aksel Puulmannef5750285438
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex d800a9b1ad..d727858fca 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -5,7 +5,13 @@ defmodule Plausible.Stats.Filters.QueryParser do@default_include %{imports: false,- time_labels: false+ time_labels: false,+ total_rows: false+ }++ @default_pagination %{+ limit: 10_000,+ offset: 0}def parse(site, schema_type, params, now \\ nil) when is_map(params) do@@ -26,6 +32,7 @@ defmodule Plausible.Stats.Filters.QueryParser do{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),{:ok, include} <- parse_include(Map.get(params, "include", %{})),+ {:ok, pagination} <- parse_pagination(Map.get(params, "pagination", %{})),preloaded_goals <- preload_goals_if_needed(site, filters, dimensions),query = %{metrics: metrics,@@ -35,7 +42,8 @@ defmodule Plausible.Stats.Filters.QueryParser doorder_by: order_by,timezone: site.timezone,preloaded_goals: preloaded_goals,- include: include+ include: include,+ pagination: pagination},:ok <- validate_order_by(query),:ok <- validate_custom_props_access(site, query),@@ -306,18 +314,15 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{i(entry)}'."}defp parse_include(include) when is_map(include) do- with {:ok, parsed_include_list} <- parse_list(include, &parse_include_value/1) do- include = Map.merge(@default_include, Enum.into(parsed_include_list, %{}))-- {:ok, include}- end+ {:ok, Map.merge(@default_include, atomize_keys(include))}end- defp parse_include_value({"imports", value}) when is_boolean(value),- do: {:ok, {:imports, value}}+ defp parse_pagination(pagination) when is_map(pagination) do+ {:ok, Map.merge(@default_pagination, atomize_keys(pagination))}+ end- defp parse_include_value({"time_labels", value}) when is_boolean(value),- do: {:ok, {:time_labels, value}}+ defp atomize_keys(map),+ do: Map.new(map, fn {key, value} -> {String.to_existing_atom(key), value} end)defp parse_filter_key_string(filter_key, error_message \\ "") docase filter_key dodiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 682e854c10..a532928c8e 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -20,9 +20,11 @@ defmodule Plausible.Stats.Query dopreloaded_goals: [],include: %{imports: false,- time_labels: false+ time_labels: false,+ total_rows: false},- debug_metadata: %{}+ debug_metadata: %{},+ pagination: nilrequire OpenTelemetry.Tracer, as: Traceralias Plausible.Stats.{Filters, Imported, Legacy}diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.exindex c332c88a0a..a946520f14 100644--- a/lib/plausible/stats/query_result.ex+++ b/lib/plausible/stats/query_result.ex@@ -26,7 +26,7 @@ defmodule Plausible.Stats.QueryResult dostruct!(__MODULE__,results: results_list,- meta: meta(query),+ meta: meta(query, results),query:Jason.OrderedObject.new(site_id: site.domain,@@ -38,7 +38,8 @@ defmodule Plausible.Stats.QueryResult dofilters: query.filters,dimensions: query.dimensions,order_by: query.order_by |> Enum.map(&Tuple.to_list/1),- include: query.include |> Map.filter(fn {_key, val} -> val end)+ include: query.include |> Map.filter(fn {_key, val} -> val end),+ pagination: query.pagination))end@@ -70,7 +71,7 @@ defmodule Plausible.Stats.QueryResult do@imports_unsupported_interval_warning "Imported stats are not included because the time dimension (i.e. the interval) is too short."- defp meta(query) do+ defp meta(query, results) do%{imports_included: if(query.include.imports, do: query.include_imported, else: nil),imports_skip_reason:@@ -82,12 +83,16 @@ defmodule Plausible.Stats.QueryResult do_ -> nilend,time_labels:- if(query.include.time_labels, do: Plausible.Stats.Time.time_labels(query), else: nil)+ if(query.include.time_labels, do: Plausible.Stats.Time.time_labels(query), else: nil),+ total_rows: if(query.include.total_rows, do: total_rows(results), else: nil)}|> Enum.reject(fn {_, value} -> is_nil(value) end)|> Enum.into(%{})end+ defp total_rows([]), do: 0+ defp total_rows([first_row | _rest]), do: first_row.total_rows+defp to_iso8601(datetime, timezone) dodatetime|> DateTime.shift_zone!(timezone)diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex dcc37f02d5..0a4e424097 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -23,6 +23,8 @@ defmodule Plausible.Stats.SQL.QueryBuilder do{event_q, event_query},{sessions_q, sessions_query})+ |> paginate(query.pagination)+ |> select_total_rows(query.include.total_rows)enddefp build_events_query(_site, %Query{metrics: []}), do: nil@@ -184,6 +186,22 @@ defmodule Plausible.Stats.SQL.QueryBuilder do|> build_order_by(events_query)end+ # NOTE: Old queries do their own pagination+ defp paginate(q, nil = _pagination), do: q++ defp paginate(q, pagination) do+ q+ |> limit(^pagination.limit)+ |> offset(^pagination.offset)+ end++ defp select_total_rows(q, false = _include_total_rows), do: q++ defp select_total_rows(q, true = _include_total_rows) do+ q+ |> select_merge([], %{total_rows: fragment("count() over ()")})+ end+def build_group_by_join(%Query{dimensions: []}), do: truedef build_group_by_join(query) dodiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 6e2a0d8fad..3b3b51b55c 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -32,7 +32,7 @@ defmodule Plausible.Stats.Timeseries dodimensions: [time_dimension(query)],order_by: [{time_dimension(query), :asc}],v2: true,- include: %{time_labels: true, imports: query.include.imports}+ include: %{time_labels: true, imports: query.include.imports, total_rows: false})|> QueryOptimizer.optimize()diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.jsonindex b6c062f334..def983a1b2 100644--- a/priv/json-schemas/query-api-schema.json+++ b/priv/json-schemas/query-api-schema.json@@ -48,6 +48,28 @@},"imports": {"type": "boolean"+ },+ "total_rows": {+ "type": "boolean",+ "default": false,+ "description": "If set, returns the total number of result rows rows before pagination under `meta.total_rows`"+ }+ }+ },+ "pagination": {+ "type": "object",+ "properties": {+ "limit": {+ "type": "integer",+ "minimum": 1,+ "maximum": 10000,+ "description": "Number of rows to limit result to."+ },+ "offset": {+ "type": "integer",+ "minimum": 0,+ "default": 0,+ "description": "Pagination offset."}}}diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex 8163ff8107..5fff611389 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -68,7 +68,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []}@@ -90,7 +91,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -131,7 +133,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []},:internal@@ -197,7 +200,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []},:internal@@ -322,7 +326,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -347,7 +352,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -373,7 +379,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -439,7 +446,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})@@ -458,7 +466,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -518,7 +527,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -565,7 +575,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -591,7 +602,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["time"],- "include" => %{"imports" => true, "time_labels" => true}+ "include" => %{"imports" => true, "time_labels" => true, "total_rows" => true}}|> check_success(site, %{metrics: [:visitors],@@ -600,7 +611,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["time"],order_by: nil,timezone: site.timezone,- include: %{imports: true, time_labels: true},+ include: %{imports: true, time_labels: true, total_rows: true},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -626,6 +638,49 @@ defmodule Plausible.Stats.Filters.QueryParserTest doendend+ describe "pagination validation" do+ test "setting pagination values", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["time"],+ "pagination" => %{"limit" => 100, "offset" => 200}+ }+ |> check_success(site, %{+ metrics: [:visitors],+ utc_time_range: @date_range_day,+ filters: [],+ dimensions: ["time"],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 100, offset: 200},+ preloaded_goals: []+ })+ end++ test "out of range limit value", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "pagination" => %{"limit" => 100_000}+ }+ |> check_error(site, "#/pagination/limit: Expected the value to be <= 10000")+ end++ test "out of range offset value", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "pagination" => %{"offset" => -5}+ }+ |> check_error(site, "#/pagination/offset: Expected the value to be >= 0")+ end+ end+describe "event:goal filter validation" dotest "valid filters", %{site: site} doinsert(:goal, %{site: site, event_name: "Signup"})@@ -652,7 +707,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: ^expected_timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: [%Plausible.Goal{page_path: "/thank-you"},%Plausible.Goal{event_name: "Signup"}@@ -924,7 +980,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:#{unquote(dimension)}"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -945,7 +1002,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["visit:#{unquote(dimension)}"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -965,7 +1023,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:props:foobar"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -1026,7 +1085,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: [{:events, :desc}, {:visitors, :asc}],timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -1046,7 +1106,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:name"],order_by: [{"event:name", :desc}],timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -1196,7 +1257,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:goal"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -1279,7 +1341,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["visit:device"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -1311,7 +1374,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:page"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})end@@ -1330,7 +1394,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false},+ include: %{imports: false, time_labels: false, total_rows: false},+ pagination: %{limit: 10_000, offset: 0},preloaded_goals: []})enddiff --git a/test/plausible/stats/query_result_test.exs b/test/plausible/stats/query_result_test.exsindex 3782a0e9c7..230a5bf645 100644--- a/test/plausible/stats/query_result_test.exs+++ b/test/plausible/stats/query_result_test.exs@@ -64,6 +64,10 @@ defmodule Plausible.Stats.QueryResultTest do],"include": {"imports": true+ },+ "pagination": {+ "offset": 0,+ "limit": 10000}}}\diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex 7d16b4f767..e83b1ff513 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -3197,15 +3197,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do)])- conn =- get(conn, "/api/v1/stats/breakdown", %{- "site_id" => site.domain,- "period" => "day",- "date" => "2021-01-01",- "property" => "event:page",- "metrics" => "bounce_rate"- })-conn =post(conn, "/api/v2/query", %{"site_id" => site.domain,@@ -3246,4 +3237,94 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert results2 == [%{"metrics" => [3], "dimensions" => []}]endend++ describe "pagination" do+ setup %{site: site} = context do+ populate_stats(site, [+ build(:pageview, pathname: "/1"),+ build(:pageview, pathname: "/2"),+ build(:pageview, pathname: "/3"),+ build(:pageview, pathname: "/4"),+ build(:pageview, pathname: "/5"),+ build(:pageview, pathname: "/6"),+ build(:pageview, pathname: "/7"),+ build(:pageview, pathname: "/8")+ ])++ context+ end++ test "pagination above total count - all results are returned", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["pageviews"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "order_by" => [["event:page", "asc"]],+ "include" => %{"total_rows" => true},+ "pagination" => %{"limit" => 10}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/1"], "metrics" => [1]},+ %{"dimensions" => ["/2"], "metrics" => [1]},+ %{"dimensions" => ["/3"], "metrics" => [1]},+ %{"dimensions" => ["/4"], "metrics" => [1]},+ %{"dimensions" => ["/5"], "metrics" => [1]},+ %{"dimensions" => ["/6"], "metrics" => [1]},+ %{"dimensions" => ["/7"], "metrics" => [1]},+ %{"dimensions" => ["/8"], "metrics" => [1]}+ ]++ assert json_response(conn, 200)["meta"]["total_rows"] == 8+ end++ test "pagination with offset", %{conn: conn, site: site} do+ query = %{+ "site_id" => site.domain,+ "metrics" => ["pageviews"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "order_by" => [["event:page", "asc"]],+ "include" => %{"total_rows" => true}+ }++ conn1 = post(conn, "/api/v2/query", Map.put(query, "pagination", %{"limit" => 3}))++ assert json_response(conn1, 200)["results"] == [+ %{"dimensions" => ["/1"], "metrics" => [1]},+ %{"dimensions" => ["/2"], "metrics" => [1]},+ %{"dimensions" => ["/3"], "metrics" => [1]}+ ]++ assert json_response(conn1, 200)["meta"]["total_rows"] == 8++ conn2 =+ post(conn, "/api/v2/query", Map.put(query, "pagination", %{"limit" => 3, "offset" => 3}))++ assert json_response(conn2, 200)["results"] == [+ %{"dimensions" => ["/4"], "metrics" => [1]},+ %{"dimensions" => ["/5"], "metrics" => [1]},+ %{"dimensions" => ["/6"], "metrics" => [1]}+ ]++ assert json_response(conn2, 200)["meta"]["total_rows"] == 8++ conn3 =+ post(conn, "/api/v2/query", Map.put(query, "pagination", %{"limit" => 3, "offset" => 6}))++ assert json_response(conn3, 200)["results"] == [+ %{"dimensions" => ["/7"], "metrics" => [1]},+ %{"dimensions" => ["/8"], "metrics" => [1]}+ ]++ assert json_response(conn3, 200)["meta"]["total_rows"] == 8++ conn4 =+ post(conn, "/api/v2/query", Map.put(query, "pagination", %{"limit" => 3, "offset" => 9}))++ assert json_response(conn4, 200)["results"] == []+ end+ endend