Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

APIv2: Implement pagination and `include.total_rows` (#4575)

Karl-Aksel Puulmannef5750285438
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex
index d800a9b1ad..d727858fca 100644
--- a/lib/plausible/stats/filters/query_parser.ex
+++ b/lib/plausible/stats/filters/query_parser.ex
@@ -5,7 +5,13 @@ defmodule Plausible.Stats.Filters.QueryParser do
@default_include %{
imports: false,
- time_labels: false
+ time_labels: false,
+ total_rows: false
+ }
+
+ @default_pagination %{
+ limit: 10_000,
+ offset: 0
}
def parse(site, schema_type, params, now \\ nil) when is_map(params) do
@@ -26,6 +32,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),
{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),
{:ok, include} <- parse_include(Map.get(params, "include", %{})),
+ {:ok, pagination} <- parse_pagination(Map.get(params, "pagination", %{})),
preloaded_goals <- preload_goals_if_needed(site, filters, dimensions),
query = %{
metrics: metrics,
@@ -35,7 +42,8 @@ defmodule Plausible.Stats.Filters.QueryParser do
order_by: order_by,
timezone: site.timezone,
preloaded_goals: preloaded_goals,
- include: include
+ include: include,
+ pagination: pagination
},
:ok <- validate_order_by(query),
:ok <- validate_custom_props_access(site, query),
@@ -306,18 +314,15 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{i(entry)}'."}
defp parse_include(include) when is_map(include) do
- with {:ok, parsed_include_list} <- parse_list(include, &parse_include_value/1) do
- include = Map.merge(@default_include, Enum.into(parsed_include_list, %{}))
-
- {:ok, include}
- end
+ {:ok, Map.merge(@default_include, atomize_keys(include))}
end
- defp parse_include_value({"imports", value}) when is_boolean(value),
- do: {:ok, {:imports, value}}
+ defp parse_pagination(pagination) when is_map(pagination) do
+ {:ok, Map.merge(@default_pagination, atomize_keys(pagination))}
+ end
- defp parse_include_value({"time_labels", value}) when is_boolean(value),
- do: {:ok, {:time_labels, value}}
+ defp atomize_keys(map),
+ do: Map.new(map, fn {key, value} -> {String.to_existing_atom(key), value} end)
defp parse_filter_key_string(filter_key, error_message \\ "") do
case filter_key do
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 682e854c10..a532928c8e 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -20,9 +20,11 @@ defmodule Plausible.Stats.Query do
preloaded_goals: [],
include: %{
imports: false,
- time_labels: false
+ time_labels: false,
+ total_rows: false
},
- debug_metadata: %{}
+ debug_metadata: %{},
+ pagination: nil
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Filters, Imported, Legacy}
diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.ex
index c332c88a0a..a946520f14 100644
--- a/lib/plausible/stats/query_result.ex
+++ b/lib/plausible/stats/query_result.ex
@@ -26,7 +26,7 @@ defmodule Plausible.Stats.QueryResult do
struct!(
__MODULE__,
results: results_list,
- meta: meta(query),
+ meta: meta(query, results),
query:
Jason.OrderedObject.new(
site_id: site.domain,
@@ -38,7 +38,8 @@ defmodule Plausible.Stats.QueryResult do
filters: query.filters,
dimensions: query.dimensions,
order_by: query.order_by |> Enum.map(&Tuple.to_list/1),
- include: query.include |> Map.filter(fn {_key, val} -> val end)
+ include: query.include |> Map.filter(fn {_key, val} -> val end),
+ pagination: query.pagination
)
)
end
@@ -70,7 +71,7 @@ defmodule Plausible.Stats.QueryResult do
@imports_unsupported_interval_warning "Imported stats are not included because the time dimension (i.e. the interval) is too short."
- defp meta(query) do
+ defp meta(query, results) do
%{
imports_included: if(query.include.imports, do: query.include_imported, else: nil),
imports_skip_reason:
@@ -82,12 +83,16 @@ defmodule Plausible.Stats.QueryResult do
_ -> nil
end,
time_labels:
- if(query.include.time_labels, do: Plausible.Stats.Time.time_labels(query), else: nil)
+ if(query.include.time_labels, do: Plausible.Stats.Time.time_labels(query), else: nil),
+ total_rows: if(query.include.total_rows, do: total_rows(results), else: nil)
}
|> Enum.reject(fn {_, value} -> is_nil(value) end)
|> Enum.into(%{})
end
+ defp total_rows([]), do: 0
+ defp total_rows([first_row | _rest]), do: first_row.total_rows
+
defp to_iso8601(datetime, timezone) do
datetime
|> DateTime.shift_zone!(timezone)
diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex
index dcc37f02d5..0a4e424097 100644
--- a/lib/plausible/stats/sql/query_builder.ex
+++ b/lib/plausible/stats/sql/query_builder.ex
@@ -23,6 +23,8 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
{event_q, event_query},
{sessions_q, sessions_query}
)
+ |> paginate(query.pagination)
+ |> select_total_rows(query.include.total_rows)
end
defp build_events_query(_site, %Query{metrics: []}), do: nil
@@ -184,6 +186,22 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|> build_order_by(events_query)
end
+ # NOTE: Old queries do their own pagination
+ defp paginate(q, nil = _pagination), do: q
+
+ defp paginate(q, pagination) do
+ q
+ |> limit(^pagination.limit)
+ |> offset(^pagination.offset)
+ end
+
+ defp select_total_rows(q, false = _include_total_rows), do: q
+
+ defp select_total_rows(q, true = _include_total_rows) do
+ q
+ |> select_merge([], %{total_rows: fragment("count() over ()")})
+ end
+
def build_group_by_join(%Query{dimensions: []}), do: true
def build_group_by_join(query) do
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index 6e2a0d8fad..3b3b51b55c 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -32,7 +32,7 @@ defmodule Plausible.Stats.Timeseries do
dimensions: [time_dimension(query)],
order_by: [{time_dimension(query), :asc}],
v2: true,
- include: %{time_labels: true, imports: query.include.imports}
+ include: %{time_labels: true, imports: query.include.imports, total_rows: false}
)
|> QueryOptimizer.optimize()
diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json
index b6c062f334..def983a1b2 100644
--- a/priv/json-schemas/query-api-schema.json
+++ b/priv/json-schemas/query-api-schema.json
@@ -48,6 +48,28 @@
},
"imports": {
"type": "boolean"
+ },
+ "total_rows": {
+ "type": "boolean",
+ "default": false,
+ "description": "If set, returns the total number of result rows rows before pagination under `meta.total_rows`"
+ }
+ }
+ },
+ "pagination": {
+ "type": "object",
+ "properties": {
+ "limit": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 10000,
+ "description": "Number of rows to limit result to."
+ },
+ "offset": {
+ "type": "integer",
+ "minimum": 0,
+ "default": 0,
+ "description": "Pagination offset."
}
}
}
diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs
index 8163ff8107..5fff611389 100644
--- a/test/plausible/stats/query_parser_test.exs
+++ b/test/plausible/stats/query_parser_test.exs
@@ -68,7 +68,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
}
@@ -90,7 +91,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -131,7 +133,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
},
:internal
@@ -197,7 +200,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
},
:internal
@@ -322,7 +326,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -347,7 +352,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -373,7 +379,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -439,7 +446,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
@@ -458,7 +466,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -518,7 +527,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -565,7 +575,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -591,7 +602,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["time"],
- "include" => %{"imports" => true, "time_labels" => true}
+ "include" => %{"imports" => true, "time_labels" => true, "total_rows" => true}
}
|> check_success(site, %{
metrics: [:visitors],
@@ -600,7 +611,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["time"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: true, time_labels: true},
+ include: %{imports: true, time_labels: true, total_rows: true},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -626,6 +638,49 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
end
end
+ describe "pagination validation" do
+ test "setting pagination values", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["time"],
+ "pagination" => %{"limit" => 100, "offset" => 200}
+ }
+ |> check_success(site, %{
+ metrics: [:visitors],
+ utc_time_range: @date_range_day,
+ filters: [],
+ dimensions: ["time"],
+ order_by: nil,
+ timezone: site.timezone,
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 100, offset: 200},
+ preloaded_goals: []
+ })
+ end
+
+ test "out of range limit value", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "pagination" => %{"limit" => 100_000}
+ }
+ |> check_error(site, "#/pagination/limit: Expected the value to be <= 10000")
+ end
+
+ test "out of range offset value", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "pagination" => %{"offset" => -5}
+ }
+ |> check_error(site, "#/pagination/offset: Expected the value to be >= 0")
+ end
+ end
+
describe "event:goal filter validation" do
test "valid filters", %{site: site} do
insert(:goal, %{site: site, event_name: "Signup"})
@@ -652,7 +707,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: ^expected_timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: [
%Plausible.Goal{page_path: "/thank-you"},
%Plausible.Goal{event_name: "Signup"}
@@ -924,7 +980,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:#{unquote(dimension)}"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -945,7 +1002,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["visit:#{unquote(dimension)}"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -965,7 +1023,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:props:foobar"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -1026,7 +1085,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: [{:events, :desc}, {:visitors, :asc}],
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -1046,7 +1106,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:name"],
order_by: [{"event:name", :desc}],
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -1196,7 +1257,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:goal"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -1279,7 +1341,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["visit:device"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -1311,7 +1374,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:page"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@@ -1330,7 +1394,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false},
+ include: %{imports: false, time_labels: false, total_rows: false},
+ pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
diff --git a/test/plausible/stats/query_result_test.exs b/test/plausible/stats/query_result_test.exs
index 3782a0e9c7..230a5bf645 100644
--- a/test/plausible/stats/query_result_test.exs
+++ b/test/plausible/stats/query_result_test.exs
@@ -64,6 +64,10 @@ defmodule Plausible.Stats.QueryResultTest do
],
"include": {
"imports": true
+ },
+ "pagination": {
+ "offset": 0,
+ "limit": 10000
}
}
}\
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
index 7d16b4f767..e83b1ff513 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
@@ -3197,15 +3197,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
)
])
- conn =
- get(conn, "/api/v1/stats/breakdown", %{
- "site_id" => site.domain,
- "period" => "day",
- "date" => "2021-01-01",
- "property" => "event:page",
- "metrics" => "bounce_rate"
- })
-
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
@@ -3246,4 +3237,94 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert results2 == [%{"metrics" => [3], "dimensions" => []}]
end
end
+
+ describe "pagination" do
+ setup %{site: site} = context do
+ populate_stats(site, [
+ build(:pageview, pathname: "/1"),
+ build(:pageview, pathname: "/2"),
+ build(:pageview, pathname: "/3"),
+ build(:pageview, pathname: "/4"),
+ build(:pageview, pathname: "/5"),
+ build(:pageview, pathname: "/6"),
+ build(:pageview, pathname: "/7"),
+ build(:pageview, pathname: "/8")
+ ])
+
+ context
+ end
+
+ test "pagination above total count - all results are returned", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["pageviews"],
+ "date_range" => "all",
+ "dimensions" => ["event:page"],
+ "order_by" => [["event:page", "asc"]],
+ "include" => %{"total_rows" => true},
+ "pagination" => %{"limit" => 10}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["/1"], "metrics" => [1]},
+ %{"dimensions" => ["/2"], "metrics" => [1]},
+ %{"dimensions" => ["/3"], "metrics" => [1]},
+ %{"dimensions" => ["/4"], "metrics" => [1]},
+ %{"dimensions" => ["/5"], "metrics" => [1]},
+ %{"dimensions" => ["/6"], "metrics" => [1]},
+ %{"dimensions" => ["/7"], "metrics" => [1]},
+ %{"dimensions" => ["/8"], "metrics" => [1]}
+ ]
+
+ assert json_response(conn, 200)["meta"]["total_rows"] == 8
+ end
+
+ test "pagination with offset", %{conn: conn, site: site} do
+ query = %{
+ "site_id" => site.domain,
+ "metrics" => ["pageviews"],
+ "date_range" => "all",
+ "dimensions" => ["event:page"],
+ "order_by" => [["event:page", "asc"]],
+ "include" => %{"total_rows" => true}
+ }
+
+ conn1 = post(conn, "/api/v2/query", Map.put(query, "pagination", %{"limit" => 3}))
+
+ assert json_response(conn1, 200)["results"] == [
+ %{"dimensions" => ["/1"], "metrics" => [1]},
+ %{"dimensions" => ["/2"], "metrics" => [1]},
+ %{"dimensions" => ["/3"], "metrics" => [1]}
+ ]
+
+ assert json_response(conn1, 200)["meta"]["total_rows"] == 8
+
+ conn2 =
+ post(conn, "/api/v2/query", Map.put(query, "pagination", %{"limit" => 3, "offset" => 3}))
+
+ assert json_response(conn2, 200)["results"] == [
+ %{"dimensions" => ["/4"], "metrics" => [1]},
+ %{"dimensions" => ["/5"], "metrics" => [1]},
+ %{"dimensions" => ["/6"], "metrics" => [1]}
+ ]
+
+ assert json_response(conn2, 200)["meta"]["total_rows"] == 8
+
+ conn3 =
+ post(conn, "/api/v2/query", Map.put(query, "pagination", %{"limit" => 3, "offset" => 6}))
+
+ assert json_response(conn3, 200)["results"] == [
+ %{"dimensions" => ["/7"], "metrics" => [1]},
+ %{"dimensions" => ["/8"], "metrics" => [1]}
+ ]
+
+ assert json_response(conn3, 200)["meta"]["total_rows"] == 8
+
+ conn4 =
+ post(conn, "/api/v2/query", Map.put(query, "pagination", %{"limit" => 3, "offset" => 9}))
+
+ assert json_response(conn4, 200)["results"] == []
+ end
+ end
end