Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
APIv2: JSON schema validation, separate internal and public API validation (#4464)
Karl-Aksel Puulmann9c71161eab48
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex a7c3c9ab04..f5f93e72f2 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -5,16 +5,19 @@ defmodule Plausible.Stats.Filters.QueryParser doalias Plausible.Stats.Filtersalias Plausible.Stats.Queryalias Plausible.Stats.Metrics+ alias Plausible.Stats.JSONSchema@default_include %{imports: false,time_labels: false}- def parse(site, params) when is_map(params) do- with {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),+ def parse(site, schema_type, params, now \\ nil) when is_map(params) do+ with :ok <- JSONSchema.validate(schema_type, params),+ {:ok, date} <- parse_date(site, Map.get(params, "date"), now),+ {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),{:ok, filters} <- parse_filters(Map.get(params, "filters", [])),- {:ok, date_range} <- parse_date_range(site, Map.take(params, ["date_range", "date"])),+ {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range"), date),{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),{:ok, include} <- parse_include(Map.get(params, "include", %{})),@@ -38,22 +41,14 @@ defmodule Plausible.Stats.Filters.QueryParser doendend- defp parse_metrics([]), do: {:error, "No valid metrics passed"}-defp parse_metrics(metrics) when is_list(metrics) do- if length(metrics) == length(Enum.uniq(metrics)) do- parse_list(metrics, &parse_metric/1)- else- {:error, "Metrics cannot be queried multiple times"}- end+ parse_list(metrics, &parse_metric/1)end- defp parse_metrics(_invalid_metrics), do: {:error, "Invalid metrics passed"}-defp parse_metric(metric_str) docase Metrics.from_string(metric_str) do{:ok, metric} -> {:ok, metric}- _ -> {:error, "Unknown metric '#{inspect(metric_str)}'"}+ _ -> {:error, "Unknown metric '#{i(metric_str)}'."}endend@@ -61,7 +56,7 @@ defmodule Plausible.Stats.Filters.QueryParser doparse_list(filters, &parse_filter/1)end- def parse_filters(_invalid_metrics), do: {:error, "Invalid filters passed"}+ def parse_filters(_invalid_metrics), do: {:error, "Invalid filters passed."}defp parse_filter(filter) dowith {:ok, operator} <- parse_operator(filter),@@ -77,13 +72,13 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_operator(["does_not_match" | _rest]), do: {:ok, :does_not_match}defp parse_operator(["contains" | _rest]), do: {:ok, :contains}defp parse_operator(["does_not_contain" | _rest]), do: {:ok, :does_not_contain}- defp parse_operator(filter), do: {:error, "Unknown operator for filter '#{inspect(filter)}'"}+ defp parse_operator(filter), do: {:error, "Unknown operator for filter '#{i(filter)}'."}defp parse_filter_key([_operator, filter_key | _rest] = filter) do- parse_filter_key_string(filter_key, "Invalid filter '#{inspect(filter)}")+ parse_filter_key_string(filter_key, "Invalid filter '#{i(filter)}")end- defp parse_filter_key(filter), do: {:error, "Invalid filter '#{inspect(filter)}'"}+ defp parse_filter_key(filter), do: {:error, "Invalid filter '#{i(filter)}'."}defp parse_filter_rest(operator, filter)when operator in [:is, :is_not, :matches, :does_not_match, :contains, :does_not_contain],@@ -102,35 +97,29 @@ defmodule Plausible.Stats.Filters.QueryParser do{:ok, [list]}else{:error,- "Invalid visit:country filter, visit:country needs to be a valid 2-letter country code"}+ "Invalid visit:country filter, visit:country needs to be a valid 2-letter country code."}end{_, true} ->{:ok, [list]}_ ->- {:error, "Invalid filter '#{inspect(filter)}'"}+ {:error, "Invalid filter '#{i(filter)}'."}endend- defp parse_clauses_list(filter), do: {:error, "Invalid filter '#{inspect(filter)}'"}+ defp parse_clauses_list(filter), do: {:error, "Invalid filter '#{i(filter)}'"}- defp parse_date_range(site, %{"date_range" => date_range, "date" => date})- when is_binary(date) do- case Date.from_iso8601(date) do- {:ok, date} -> parse_date_range(site, date_range, date)- _ -> {:error, "Invalid date '#{date}'"}- end- end+ defp parse_date(site, nil, nil), do: {:ok, today(site)}- defp parse_date_range(site, %{"date_range" => date_range, "date" => date})- when is_struct(date) do- parse_date_range(site, date_range, date)+ defp parse_date(_site, date_string, nil) when is_binary(date_string) do+ case Date.from_iso8601(date_string) do+ {:ok, date} -> {:ok, date}+ _ -> {:error, "Invalid date '#{date_string}'."}+ endend- defp parse_date_range(site, %{"date_range" => date_range}) do- parse_date_range(site, date_range, today(site))- end+ defp parse_date(_site, _date_string, param_date), do: {:ok, param_date}defp parse_date_range(_site, "day", date) do{:ok, Date.range(date, date)}@@ -192,34 +181,28 @@ defmodule Plausible.Stats.Filters.QueryParser do{:ok, to_date} <- Date.from_iso8601(to_date_string) do{:ok, Date.range(from_date, to_date)}else- _ -> {:error, "Invalid date_range '#{inspect([from_date_string, to_date_string])}'"}+ _ -> {:error, "Invalid date_range '#{i([from_date_string, to_date_string])}'."}endenddefp parse_date_range(_site, unknown, _),- do: {:error, "Invalid date_range '#{inspect(unknown)}'"}+ do: {:error, "Invalid date_range '#{i(unknown)}'."}defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date()defp parse_dimensions(dimensions) when is_list(dimensions) do- if length(dimensions) == length(Enum.uniq(dimensions)) do- parse_list(- dimensions,- &parse_dimension_entry(&1, "Invalid dimensions '#{inspect(dimensions)}'")- )- else- {:error, "Some dimensions are listed multiple times"}- end+ parse_list(+ dimensions,+ &parse_dimension_entry(&1, "Invalid dimensions '#{i(dimensions)}'")+ )end- defp parse_dimensions(dimensions), do: {:error, "Invalid dimensions '#{inspect(dimensions)}'"}-defp parse_order_by(order_by) when is_list(order_by) doparse_list(order_by, &parse_order_by_entry/1)enddefp parse_order_by(nil), do: {:ok, nil}- defp parse_order_by(order_by), do: {:error, "Invalid order_by '#{inspect(order_by)}'"}+ defp parse_order_by(order_by), do: {:error, "Invalid order_by '#{i(order_by)}'."}defp parse_order_by_entry(entry) dowith {:ok, value} <- parse_metric_or_dimension(entry),@@ -248,7 +231,7 @@ defmodule Plausible.Stats.Filters.QueryParser do{{:ok, time}, _, _} -> {:ok, time}{_, {:ok, metric}, _} -> {:ok, metric}{_, _, {:ok, dimension}} -> {:ok, dimension}- _ -> {:error, "Invalid order_by entry '#{inspect(entry)}'"}+ _ -> {:error, "Invalid order_by entry '#{i(entry)}'."}endend@@ -261,7 +244,7 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_order_direction([_, "asc"]), do: {:ok, :asc}defp parse_order_direction([_, "desc"]), do: {:ok, :desc}- defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{inspect(entry)}'"}+ defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{i(entry)}'."}defp parse_include(include) when is_map(include) dowith {:ok, parsed_include_list} <- parse_list(include, &parse_include_value/1) do@@ -271,17 +254,12 @@ defmodule Plausible.Stats.Filters.QueryParser doendend- defp parse_include(entry), do: {:error, "Invalid include passed '#{inspect(entry)}'"}-defp parse_include_value({"imports", value}) when is_boolean(value),do: {:ok, {:imports, value}}defp parse_include_value({"time_labels", value}) when is_boolean(value),do: {:ok, {:time_labels, value}}- defp parse_include_value({key, value}),- do: {:error, "Invalid include entry '#{inspect(%{key => value})}'"}-defp parse_filter_key_string(filter_key, error_message \\ "") docase filter_key do"event:props:" <> property_name ->@@ -325,7 +303,7 @@ defmodule Plausible.Stats.Filters.QueryParser do_ ->{:error,- "Invalid order_by entry '#{inspect(invalid_entry)}'. Entry is not a queried metric or dimension"}+ "Invalid order_by entry '#{i(invalid_entry)}'. Entry is not a queried metric or dimension."}endelse:ok@@ -390,7 +368,7 @@ defmodule Plausible.Stats.Filters.QueryParser doif valid? do:okelse- {:error, "The owner of this site does not have access to the custom properties feature"}+ {:error, "The owner of this site does not have access to the custom properties feature."}endend@@ -405,17 +383,17 @@ defmodule Plausible.Stats.Filters.QueryParser donot is_nil(Query.get_filter(query, "event:goal")) do:okelse- {:error, "Metric `#{metric}` can only be queried with event:goal filters or dimensions"}+ {:error, "Metric `#{metric}` can only be queried with event:goal filters or dimensions."}endenddefp validate_metric(:views_per_visit = metric, query) docond donot is_nil(Query.get_filter(query, "event:page")) ->- {:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`"}+ {:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."}length(query.dimensions) > 0 ->- {:error, "Metric `#{metric}` cannot be queried with `dimensions`"}+ {:error, "Metric `#{metric}` cannot be queried with `dimensions`."}true ->:ok@@ -433,7 +411,7 @@ defmodule Plausible.Stats.Filters.QueryParser do:okelse{:error,- "Session metric(s) `#{sessions_metrics |> Enum.join(", ")}` cannot be queried along with event dimensions"}+ "Session metric(s) `#{sessions_metrics |> Enum.join(", ")}` cannot be queried along with event dimensions."}endend@@ -449,12 +427,14 @@ defmodule Plausible.Stats.Filters.QueryParser dotime_dimension? = Enum.any?(query.dimensions, &String.starts_with?(&1, "time"))if query.include.time_labels and not time_dimension? do- {:error, "Invalid include.time_labels: requires a time dimension"}+ {:error, "Invalid include.time_labels: requires a time dimension."}else:okendend+ defp i(value), do: inspect(value, charlists: :as_lists)+defp parse_list(list, parser_function) doEnum.reduce_while(list, {:ok, []}, fn value, {:ok, results} ->case parser_function.(value) dodiff --git a/lib/plausible/stats/json_schema.ex b/lib/plausible/stats/json_schema.exnew file mode 100644index 0000000000..08c22467a6--- /dev/null+++ b/lib/plausible/stats/json_schema.ex@@ -0,0 +1,67 @@+defmodule Plausible.Stats.JSONSchema do+ @moduledoc """+ Module for validating query parameters against JSON schema.++ Note that `internal` queries expose some metrics, filter types and other features not+ available on the public API.+ """++ @external_resource "priv/json-schemas/query-api-schema.json"++ @raw_public_schema Application.app_dir(:plausible, "priv/json-schemas/query-api-schema.json")+ |> File.read!()+ |> Jason.decode!()++ @public_query_schema ExJsonSchema.Schema.resolve(@raw_public_schema)++ @internal_query_schema @raw_public_schema+ # Add overrides for things allowed in the internal API+ |> JSONPointer.add!(+ "#/definitions/filter_entry/oneOf/0/items/0/enum/0",+ "matches"+ )+ |> JSONPointer.add!(+ "#/definitions/filter_entry/oneOf/0/items/0/enum/0",+ "does_not_match"+ )+ |> JSONPointer.add!("#/definitions/metric/oneOf/0", %{+ "const" => "time_on_page"+ })+ |> ExJsonSchema.Schema.resolve()++ def validate(schema_type, params) do+ case ExJsonSchema.Validator.validate(schema(schema_type), params) do+ :ok -> :ok+ {:error, errors} -> {:error, format_errors(errors, params)}+ end+ end++ def raw_public_schema(), do: @raw_public_schema++ defp schema(:public), do: @public_query_schema+ defp schema(:internal), do: @internal_query_schema++ defp format_errors(errors, params) do+ errors+ |> Enum.map_join("\n", fn {error, path} ->+ value = JSONPointer.get!(params, path)++ "#{path}: #{reword(path, error, value)}"+ end)+ end++ @no_matches "Expected exactly one of the schemata to match, but none of them did."++ defp reword("#/dimensions/" <> _, @no_matches, value), do: "Invalid dimension #{i(value)}"+ defp reword("#/metrics/" <> _, @no_matches, value), do: "Invalid metric #{i(value)}"+ defp reword("#/filters/" <> _, @no_matches, value), do: "Invalid filter #{i(value)}"+ defp reword("#/date_range", @no_matches, value), do: "Invalid date range #{i(value)}"++ defp reword("#/order_by/" <> _, @no_matches, value) do+ "Invalid value in order_by #{i(value)}"+ end++ defp reword(_path, error, _value), do: error++ defp i(value), do: inspect(value, charlists: :as_lists)+enddiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 5bfcbf7c99..cbbf6ae31f 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -29,8 +29,8 @@ defmodule Plausible.Stats.Query do@type t :: %__MODULE__{}- def build(site, params, debug_metadata) do- with {:ok, query_data} <- Filters.QueryParser.parse(site, params) do+ def build(site, schema_type, params, debug_metadata) do+ with {:ok, query_data} <- Filters.QueryParser.parse(site, schema_type, params) doquery =struct!(__MODULE__, Map.to_list(query_data))|> put_imported_opts(site, %{})diff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.exindex 5b88d39e7c..dabf5daf3f 100644--- a/lib/plausible_web/controllers/api/external_query_api_controller.ex+++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex@@ -9,7 +9,7 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController dodef query(conn, params) dosite = Repo.preload(conn.assigns.site, :owner)- case Query.build(site, params, debug_metadata(conn)) do+ case Query.build(site, :public, params, debug_metadata(conn)) do{:ok, query} ->results = Plausible.Stats.query(site, query)json(conn, results)@@ -20,4 +20,8 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController do|> json(%{error: message})endend++ def schema(conn, _params) do+ json(conn, Plausible.Stats.JSONSchema.raw_public_schema())+ endenddiff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex 7ee9cc6405..3a1c6f4814 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -182,9 +182,13 @@ defmodule PlausibleWeb.Router doendscope "/api/docs", PlausibleWeb.Api do- pipe_through :internal_stats_api+ get "/query/schema.json", ExternalQueryApiController, :schema- post "/query", ExternalQueryApiController, :query+ scope assigns: %{} do+ pipe_through :internal_stats_api++ post "/query", ExternalQueryApiController, :query+ endendon_ee dodiff --git a/mix.exs b/mix.exsindex bf04deccaf..e03ef6da5f 100644--- a/mix.exs+++ b/mix.exs@@ -144,7 +144,9 @@ defmodule Plausible.MixProject do{:zstream, "~> 0.6.4"},{:con_cache, "~> 1.1.0"},{:req, "~> 0.5.0"},- {:happy_tcp, github: "ruslandoga/happy_tcp", only: [:ce, :ce_dev, :ce_test]}+ {:happy_tcp, github: "ruslandoga/happy_tcp", only: [:ce, :ce_dev, :ce_test]},+ {:ex_json_schema, "~> 0.10.2"},+ {:odgn_json_pointer, "~> 3.0.1"}]enddiff --git a/mix.lock b/mix.lockindex e4f364a518..12da3ea4a0 100644--- a/mix.lock+++ b/mix.lock@@ -47,6 +47,7 @@"ex_cldr_numbers": {:hex, :ex_cldr_numbers, "2.33.1", "49dc6e77e6d9ad22660aaa2480a7408ad3aedfbe517e4e83e5fe3a7bf5345770", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:digital_token, "~> 0.3 or ~> 1.0", [hex: :digital_token, repo: "hexpm", optional: false]}, {:ex_cldr, "~> 2.38", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:ex_cldr_currencies, "~> 2.16", [hex: :ex_cldr_currencies, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "c003bfaa3fdee6bab5195f128b94038c2ce1cf4879a759eef431dd075d9a5dac"},"ex_doc": {:hex, :ex_doc, "0.31.1", "8a2355ac42b1cc7b2379da9e40243f2670143721dd50748bf6c3b1184dae2089", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "3178c3a407c557d8343479e1ff117a96fd31bafe52a039079593fb0524ef61b0"},"ex_json_logger": {:hex, :ex_json_logger, "1.4.0", "ad1dcc1cfe6940ee1d9d489b20757c89769626ce34c4957548d6fbe155cd96f1", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "7548a1ecba290746e06214d2b3d8783c76760c779a8903a8e44bfd23a7340444"},+ "ex_json_schema": {:hex, :ex_json_schema, "0.10.2", "7c4b8c1481fdeb1741e2ce66223976edfb9bccebc8014f6aec35d4efe964fb71", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "37f43be60f8407659d4d0155a7e45e7f406dab1f827051d3d35858a709baf6a6"},"ex_machina": {:hex, :ex_machina, "2.7.0", "b792cc3127fd0680fecdb6299235b4727a4944a09ff0fa904cc639272cd92dc7", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "419aa7a39bde11894c87a615c4ecaa52d8f107bbdd81d810465186f783245bf8"},"ex_money": {:hex, :ex_money, "5.17.0", "9064a30d877d85b3e3ec7ca52339542037473bc71fc5c5b9f6c31d86516002b9", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ex_cldr_numbers, "~> 2.33", [hex: :ex_cldr_numbers, repo: "hexpm", optional: false]}, {:gringotts, "~> 1.1", [hex: :gringotts, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.0 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm", "226aa8906c85cb121f1d0ead0b108b259ba68d92c8fc79fa1758d520ff5c84c0"},"excoveralls": {:hex, :excoveralls, "0.18.0", "b92497e69465dc51bc37a6422226ee690ab437e4c06877e836f1c18daeb35da9", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "1109bb911f3cb583401760be49c02cbbd16aed66ea9509fc5479335d284da60b"},@@ -96,6 +97,7 @@"oban": {:hex, :oban, "2.17.2", "bcd1276473d8635475076b01032c00474f9c7841d3a2ca46ead26e1ec023cdd3", [:mix], [{:ecto_sql, "~> 3.6", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "de90489c05c039a6d942fa54d8fa13b858db315da2178e4e2c35c82c0a3ab556"},"observer_cli": {:hex, :observer_cli, "1.7.4", "3c1bfb6d91bf68f6a3d15f46ae20da0f7740d363ee5bc041191ce8722a6c4fae", [:mix, :rebar3], [{:recon, "~> 2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "50de6d95d814f447458bd5d72666a74624eddb0ef98bdcee61a0153aae0865ff"},"octo_fetch": {:hex, :octo_fetch, "0.3.0", "89ff501d2ac0448556ff1931634a538fe6d6cd358ba827ce1747e6a42a46efbf", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "c07e44f2214ab153743b7b3182f380798d0b294b1f283811c1e30cff64096d3d"},+ "odgn_json_pointer": {:hex, :odgn_json_pointer, "3.0.1", "0776babea6dc2e87f4569f24aaf771ea06dc62f1dd217f015bdfca550e6fcc7f", [:mix], [], "hexpm", "776d70ac887531d9235449f88b9c74a94cfa6669032bbf92e51a4c53a25212ad"},"open_api_spex": {:hex, :open_api_spex, "3.18.1", "0a73cd5dbcba7d32952dd9738c6819892933d9bae1642f04c9f200281524dd31", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "f52933cddecca675e42ead660379ae2d3853f57f5a35d201eaed85e2e81517d1"},"opentelemetry": {:hex, :opentelemetry, "1.3.1", "f0a342a74379e3540a634e7047967733da4bc8b873ec9026e224b2bd7369b1fc", [:rebar3], [{:opentelemetry_api, "~> 1.2.2", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "de476b2ac4faad3e3fe3d6e18b35dec9cb338c3b9910c2ce9317836dacad3483"},"opentelemetry_api": {:hex, :opentelemetry_api, "1.2.2", "693f47b0d8c76da2095fe858204cfd6350c27fe85d00e4b763deecc9588cf27a", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "dc77b9a00f137a858e60a852f14007bb66eda1ffbeb6c05d5fe6c9e678b05e9d"},diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.jsonnew file mode 100644index 0000000000..007575ae85--- /dev/null+++ b/priv/json-schemas/query-api-schema.json@@ -0,0 +1,330 @@+{+ "$schema": "http://json-schema.org/draft-07/schema#",+ "type": "object",+ "properties": {+ "site_id": {+ "type": "string",+ "description": "Domain of site to query"+ },+ "metrics": {+ "type": "array",+ "items": {+ "$ref": "#/definitions/metric"+ },+ "minItems": 1,+ "uniqueItems": true,+ "description": "List of metrics to query"+ },+ "date_range": {+ "$ref": "#/definitions/date_range",+ "description": "Date range to query"+ },+ "dimensions": {+ "type": "array",+ "items": {+ "$ref": "#/definitions/dimensions"+ },+ "description": "What to group the results by. Same as `property` in Plausible API v1",+ "uniqueItems": true+ },+ "filters": {+ "type": "array",+ "items": {+ "$ref": "#/definitions/filter_entry"+ },+ "description": "How to drill into your data"+ },+ "order_by": {+ "type": "array",+ "items": {+ "$ref": "#/definitions/order_by_entry"+ },+ "description": "How to order query results"+ },+ "include": {+ "type": "object",+ "properties": {+ "time_labels": {+ "type": "boolean",+ "markdownDescription": "Requires a `time` dimension. If enabled, returns each time bucket under `meta.time_labels`"+ },+ "imports": {+ "type": "boolean"+ }+ }+ }+ },+ "required": [+ "site_id",+ "metrics",+ "date_range"+ ],+ "additionalProperties": false,+ "definitions": {+ "date_range": {+ "oneOf": [+ {+ "const": "all",+ "description": "Since the start of stats in Plausible"+ },+ {+ "const": "day",+ "description": "Last day"+ },+ {+ "const": "7d",+ "description": "Last 7 days relative to today"+ },+ {+ "const": "30d",+ "description": "Last 30 days relative to today"+ },+ {+ "const": "month",+ "description": "Current calendar month"+ },+ {+ "const": "6mo",+ "description": "Last 6 calendar months relative to this month"+ },+ {+ "const": "12mo",+ "description": "Last 12 calendar months relative to this month"+ },+ {+ "const": "year",+ "description": "Since the start of this year"+ },+ {+ "type": "array",+ "items": {+ "type": "string",+ "pattern": "^\\d{4}-\\d{2}-\\d{2}$"+ },+ "markdownDescription": "Start and end dates of custom date range. Both dates are in format `YYYY-MM-DD`",+ "examples": [+ [+ "2024-01-01",+ "2024-01-31"+ ]+ ],+ "minItems": 2,+ "maxItems": 2+ }+ ]+ },+ "metric": {+ "oneOf": [+ {+ "const": "visitors",+ "description": "Metric counting the number of unique visitors"+ },+ {+ "const": "visits",+ "description": "Metric counting the number of visits/sessions"+ },+ {+ "const": "pageviews",+ "markdownDescription": "Metric counting the number of `pageview` events"+ },+ {+ "const": "views_per_visit",+ "description": "Metric for the number of pageviews divided by the number of visits. Returns a floating point number."+ },+ {+ "const": "bounce_rate",+ "description": "Bounce rate percentage"+ },+ {+ "const": "visit_duration",+ "description": "Visit duration in seconds"+ },+ {+ "const": "events",+ "markdownDescription": "The number of events (pageviews + custom events). When filtering by a goal, this metric corresponds to `Total Conversions` in the dashboard."+ },+ {+ "const": "percentage",+ "markdownDescription": "The percentage of visitors of total who fall into this category. Requires `dimensions` not to be empty"+ },+ {+ "const": "conversion_rate",+ "markdownDescription": "The percentage of visitors who completed the goal. Requires: `dimension` list passed, an `event:goal` filter or `event:goal` dimension"+ },+ {+ "const": "group_conversion_rate",+ "markdownDescription": "The percentage of visitors who completed the goal with the same dimension. Requires: dimension list passed, an `event:goal` filter or `event:goal` dimension"+ }+ ]+ },+ "simple_filter_dimensions": {+ "type": "string",+ "enum": [+ "event:name",+ "event:page",+ "event:hostname",+ "visit:source",+ "visit:referrer",+ "visit:utm_medium",+ "visit:utm_source",+ "visit:utm_campaign",+ "visit:utm_content",+ "visit:utm_term",+ "visit:screen",+ "visit:device",+ "visit:browser",+ "visit:browser_version",+ "visit:os",+ "visit:os_version",+ "visit:country",+ "visit:region",+ "visit:city",+ "visit:country_name",+ "visit:region_name",+ "visit:city_name",+ "visit:entry_page",+ "visit:exit_page",+ "visit:entry_page_hostname",+ "visit:exit_page_hostname"+ ]+ },+ "custom_property_filter_dimensions": {+ "type": "string",+ "pattern": "^event:props:.+",+ "markdownDescription": "Custom property. See [documentation](https://plausible.io/docs/custom-props/introduction) for more information",+ "examples": [+ "event:props:url",+ "event:props:path"+ ]+ },+ "goal_dimension": {+ "const": "event:goal",+ "markdownDescription": "Goal dimension"+ },+ "time_dimensions": {+ "type": "string",+ "enum": [+ "time",+ "time:month",+ "time:week",+ "time:day",+ "time:hour"+ ]+ },+ "dimensions": {+ "oneOf": [+ {+ "$ref": "#/definitions/simple_filter_dimensions"+ },+ {+ "$ref": "#/definitions/custom_property_filter_dimensions"+ },+ {+ "$ref": "#/definitions/goal_dimension"+ },+ {+ "$ref": "#/definitions/time_dimensions"+ }+ ]+ },+ "clauses": {+ "type": "array",+ "items": {+ "type": ["string", "integer"]+ }+ },+ "filter_entry": {+ "oneOf": [+ {+ "type": "array",+ "items": [+ {+ "type": "string",+ "enum": [+ "is_not",+ "does_not_contain"+ ],+ "description": "filter operation"+ },+ {+ "oneOf": [+ {+ "$ref": "#/definitions/simple_filter_dimensions"+ },+ {+ "$ref": "#/definitions/custom_property_filter_dimensions"+ }+ ]+ },+ {+ "$ref": "#/definitions/clauses"+ }+ ],+ "minItems": 3,+ "maxItems": 3+ },+ {+ "type": "array",+ "items": [+ {+ "type": "string",+ "enum": [+ "is",+ "contains"+ ],+ "description": "filter operation"+ },+ {+ "oneOf": [+ {+ "$ref": "#/definitions/goal_dimension"+ },+ {+ "$ref": "#/definitions/simple_filter_dimensions"+ },+ {+ "$ref": "#/definitions/custom_property_filter_dimensions"+ }+ ]+ },+ {+ "$ref": "#/definitions/clauses"+ }+ ],+ "minItems": 3,+ "maxItems": 3+ }+ ]+ },+ "order_by_entry": {+ "type": "array",+ "items": [+ {+ "oneOf": [+ {+ "$ref": "#/definitions/metric"+ },+ {+ "$ref": "#/definitions/simple_filter_dimensions"+ },+ {+ "$ref": "#/definitions/custom_property_filter_dimensions"+ },+ {+ "$ref": "#/definitions/time_dimensions"+ }+ ],+ "markdownDescription": "Metric or dimension to order by. Must be listed under `metrics` or `dimensions`"+ },+ {+ "type": "string",+ "enum": [+ "asc",+ "desc"+ ],+ "description": "Sorting order"+ }+ ]+ }+ }+}diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex 8ab002eb6f..062abe1cd2 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -9,17 +9,23 @@ defmodule Plausible.Stats.Filters.QueryParserTest do@today ~D[2021-05-05]@date_range Date.range(@today, @today)- def check_success(params, site, expected_result) do- assert {:ok, ^expected_result} = parse(site, Map.put(params, "date", @today))+ def check_success(params, site, expected_result),+ do: check_success(params, site, :public, expected_result)++ def check_success(params, site, schema_type, expected_result) do+ assert {:ok, ^expected_result} = parse(site, schema_type, params, @today)end- def check_error(params, site, expected_error_message) do- {:error, message} = parse(site, Map.put(params, "date", @today))- assert message =~ expected_error_message+ def check_error(params, site, expected_error_message),+ do: check_error(params, site, :public, expected_error_message)++ def check_error(params, site, schema_type, expected_error_message) do+ {:error, message} = parse(site, schema_type, params, @today)+ assert message == expected_error_messageenddef check_date_range(date_range, site, expected_date_range) do- %{"metrics" => ["visitors", "events"], "date_range" => date_range}+ %{"site_id" => site.domain, "metrics" => ["visitors", "events"], "date_range" => date_range}|> check_success(site, %{metrics: [:visitors, :events],date_range: expected_date_range,@@ -34,12 +40,12 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "parsing empty map fails", %{site: site} do%{}- |> check_error(site, "No valid metrics passed")+ |> check_error(site, "#: Required properties site_id, metrics, date_range were not present.")enddescribe "metrics validation" dotest "valid metrics passed", %{site: site} do- %{"metrics" => ["visitors", "events"], "date_range" => "all"}+ %{"site_id" => site.domain, "metrics" => ["visitors", "events"], "date_range" => "all"}|> check_success(site, %{metrics: [:visitors, :events],date_range: @date_range,@@ -53,12 +59,13 @@ defmodule Plausible.Stats.Filters.QueryParserTest doendtest "invalid metric passed", %{site: site} do- %{"metrics" => ["visitors", "event:name"], "date_range" => "all"}- |> check_error(site, "Unknown metric '\"event:name\"'")+ %{"site_id" => site.domain, "metrics" => ["visitors", "event:name"], "date_range" => "all"}+ |> check_error(site, "#/metrics/1: Invalid metric \"event:name\"")endtest "fuller list of metrics", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["time_on_page","visitors",@@ -70,7 +77,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do],"date_range" => "all"}- |> check_success(site, %{+ |> check_success(site, :internal, %{metrics: [:time_on_page,:visitors,@@ -90,23 +97,46 @@ defmodule Plausible.Stats.Filters.QueryParserTest do})end+ test "time_on_page is not a valid metric in public API", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => "all"+ }+ |> check_error(site, :public, "#/metrics/0: Invalid metric \"time_on_page\"")+ end+test "same metric queried multiple times", %{site: site} do- %{"metrics" => ["events", "visitors", "visitors"], "date_range" => "all"}- |> check_error(site, ~r/Metrics cannot be queried multiple times/)+ %{+ "site_id" => site.domain,+ "metrics" => ["events", "visitors", "visitors"],+ "date_range" => "all"+ }+ |> check_error(site, "#/metrics: Expected items to be unique but they were not.")+ end++ test "no metrics passed", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => [],+ "date_range" => "all"+ }+ |> check_error(site, "#/metrics: Expected a minimum of 1 items but got 0.")endenddescribe "filters validation" do- for operation <- [:is, :is_not, :matches, :does_not_match] do+ for operation <- [:is, :is_not, :matches, :does_not_match, :contains, :does_not_contain] dotest "#{operation} filter", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [[Atom.to_string(unquote(operation)), "event:name", ["foo"]]]}- |> check_success(site, %{+ |> check_success(site, :internal, %{metrics: [:visitors],date_range: @date_range,filters: [@@ -122,29 +152,54 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "#{operation} filter with invalid clause", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [[Atom.to_string(unquote(operation)), "event:name", "foo"]]}- |> check_error(site, ~r/Invalid filter/)+ |> check_error(+ site,+ :internal,+ "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", \"foo\"]"+ )+ end+ end++ for operation <- [:matches, :does_not_match] do+ test "#{operation} is not a valid filter operation in public API", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [+ [Atom.to_string(unquote(operation)), "event:name", ["foo"]]+ ]+ }+ |> check_error(+ site,+ :public,+ "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", [\"foo\"]]"+ )endendtest "filtering by invalid operation", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [["exists?", "event:name", ["foo"]]]}- |> check_error(site, ~r/Unknown operator for filter/)+ |> check_error(site, "#/filters/0: Invalid filter [\"exists?\", \"event:name\", [\"foo\"]]")endtest "filtering by custom properties", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [@@ -169,6 +224,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest doif dimension != "goal" dotest "filtering by event:#{dimension} filter", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [@@ -194,6 +250,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dofor dimension <- Filters.visit_props() dotest "filtering by visit:#{dimension} filter", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [@@ -217,46 +274,51 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "invalid event filter", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [["is", "event:device", ["foo"]]]}- |> check_error(site, ~r/Invalid filter /)+ |> check_error(site, "#/filters/0: Invalid filter [\"is\", \"event:device\", [\"foo\"]]")endtest "invalid visit filter", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [["is", "visit:name", ["foo"]]]}- |> check_error(site, ~r/Invalid filter /)+ |> check_error(site, "#/filters/0: Invalid filter [\"is\", \"visit:name\", [\"foo\"]]")endtest "invalid filter", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => "foobar"}- |> check_error(site, ~r/Invalid filters passed/)+ |> check_error(site, "#/filters: Type mismatch. Expected Array but got String.")endtest "numeric filter is invalid", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [["is", "visit:os_version", [123]]]}- |> check_error(site, ~r/Invalid filter /)+ |> check_error(site, "Invalid filter '[\"is\", \"visit:os_version\", [123]]'.")endtest "numbers and strings are valid for visit:city", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [["is", "visit:city", [123, 456]]]@@ -275,6 +337,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do})%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [["is", "visit:city", ["123", "456"]]]@@ -295,13 +358,14 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "invalid visit:country filter", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [["is", "visit:country", ["USA"]]]}|> check_error(site,- ~r/Invalid visit:country filter, visit:country needs to be a valid 2-letter country code/+ "Invalid visit:country filter, visit:country needs to be a valid 2-letter country code.")endend@@ -309,6 +373,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodescribe "include validation" dotest "setting include values", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["time"],@@ -328,20 +393,22 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "setting invalid imports value", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","include" => "foobar"}- |> check_error(site, ~r/Invalid include passed/)+ |> check_error(site, "#/include: Type mismatch. Expected Object but got String.")endtest "setting include.time_labels without time dimension", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","include" => %{"time_labels" => true}}- |> check_error(site, ~r/Invalid include.time_labels: requires a time dimension/)+ |> check_error(site, "Invalid include.time_labels: requires a time dimension.")endend@@ -351,6 +418,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest doinsert(:goal, %{site: site, page_path: "/thank-you"})params = %{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [@@ -358,7 +426,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do]}- assert {:ok, res} = parse(site, Map.put(params, "date", @today))+ assert {:ok, res} = parse(site, :public, params, @today)expected_timezone = site.timezoneassert %{@@ -380,24 +448,32 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "invalid event filter", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [["is", "event:goal", ["Signup"]]]}- |> check_error(site, ~r/The goal `Signup` is not configured for this site/)+ |> check_error(+ site,+ "The goal `Signup` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals"+ )endtest "invalid page filter", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [["is", "event:goal", ["Visit /thank-you"]]]}- |> check_error(site, ~r/The goal `Visit \/thank-you` is not configured for this site/)+ |> check_error(+ site,+ "The goal `Visit /thank-you` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals"+ )endend@@ -432,11 +508,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest doendtest "parsing invalid custom date range", %{site: site} do- %{"date_range" => "foo", "metrics" => ["visitors"]}- |> check_error(site, ~r/Invalid date_range '\"foo\"'/)+ %{"site_id" => site.domain, "date_range" => "foo", "metrics" => ["visitors"]}+ |> check_error(site, "#/date_range: Invalid date range \"foo\"")- %{"date_range" => ["21415-00", "eee"], "metrics" => ["visitors"]}- |> check_error(site, ~r/Invalid date_range /)+ %{"site_id" => site.domain, "date_range" => ["21415-00", "eee"], "metrics" => ["visitors"]}+ |> check_error(site, "#/date_range: Invalid date range [\"21415-00\", \"eee\"]")endend@@ -444,6 +520,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dofor dimension <- Filters.event_props() dotest "event:#{dimension} dimension", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["event:#{unquote(dimension)}"]@@ -464,6 +541,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dofor dimension <- Filters.visit_props() dotest "visit:#{dimension} dimension", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["visit:#{unquote(dimension)}"]@@ -483,6 +561,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "custom properties dimension", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["event:props:foobar"]@@ -501,44 +580,49 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "invalid custom property dimension", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["event:props:"]}- |> check_error(site, ~r/Invalid dimensions/)+ |> check_error(site, "#/dimensions/0: Invalid dimension \"event:props:\"")endtest "invalid dimension name passed", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["visitors"]}- |> check_error(site, ~r/Invalid dimensions/)+ |> check_error(site, "#/dimensions/0: Invalid dimension \"visitors\"")endtest "invalid dimension", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","dimensions" => "foobar"}- |> check_error(site, ~r/Invalid dimensions/)+ |> check_error(site, "#/dimensions: Type mismatch. Expected Array but got String.")endtest "dimensions are not unique", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["event:name", "event:name"]}- |> check_error(site, ~r/Some dimensions are listed multiple times/)+ |> check_error(site, "#/dimensions: Expected items to be unique but they were not.")endenddescribe "order_by validation" dotest "ordering by metric", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors", "events"],"date_range" => "all","order_by" => [["events", "desc"], ["visitors", "asc"]]@@ -557,6 +641,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "ordering by dimension", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["event:name"],@@ -576,29 +661,38 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "ordering by invalid value", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","order_by" => [["visssss", "desc"]]}- |> check_error(site, ~r/Invalid order_by entry/)+ |> check_error(site, "#/order_by/0/0: Invalid value in order_by \"visssss\"")endtest "ordering by not queried metric", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","order_by" => [["events", "desc"]]}- |> check_error(site, ~r/Entry is not a queried metric or dimension/)+ |> check_error(+ site,+ "Invalid order_by entry '{:events, :desc}'. Entry is not a queried metric or dimension."+ )endtest "ordering by not queried dimension", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","order_by" => [["event:name", "desc"]]}- |> check_error(site, ~r/Entry is not a queried metric or dimension/)+ |> check_error(+ site,+ "Invalid order_by entry '{\"event:name\", :desc}'. Entry is not a queried metric or dimension."+ )endend@@ -610,13 +704,14 @@ defmodule Plausible.Stats.Filters.QueryParserTest doinsert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","filters" => [["is", "event:props:foobar", ["foo"]]]}|> check_error(site,- ~r/The owner of this site does not have access to the custom properties feature/+ "The owner of this site does not have access to the custom properties feature.")end@@ -627,13 +722,14 @@ defmodule Plausible.Stats.Filters.QueryParserTest doinsert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)%{+ "site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["event:props:foobar"]}|> check_error(site,- ~r/The owner of this site does not have access to the custom properties feature/+ "The owner of this site does not have access to the custom properties feature.")endend@@ -641,12 +737,13 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodescribe "conversion_rate metric" dotest "fails validation on its own", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["conversion_rate"],"date_range" => "all"}|> check_error(site,- ~r/Metric `conversion_rate` can only be queried with event:goal filters or dimensions/+ "Metric `conversion_rate` can only be queried with event:goal filters or dimensions.")end@@ -714,25 +811,27 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "fails validation if event:page filter specified", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["views_per_visit"],"date_range" => "all","filters" => [["is", "event:page", ["/"]]]}|> check_error(site,- ~r/Metric `views_per_visit` cannot be queried with a filter on `event:page`/+ "Metric `views_per_visit` cannot be queried with a filter on `event:page`.")endtest "fails validation with dimensions", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["views_per_visit"],"date_range" => "all","dimensions" => ["event:name"]}|> check_error(site,- ~r/Metric `views_per_visit` cannot be queried with `dimensions`/+ "Metric `views_per_visit` cannot be queried with `dimensions`.")endend@@ -740,6 +839,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodescribe "session metrics" dotest "single session metric succeeds", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["bounce_rate"],"date_range" => "all","dimensions" => ["visit:device"]@@ -758,18 +858,20 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "fails if using session metric with event dimension", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["bounce_rate"],"date_range" => "all","dimensions" => ["event:props:foo"]}|> check_error(site,- "Session metric(s) `bounce_rate` cannot be queried along with event dimensions"+ "Session metric(s) `bounce_rate` cannot be queried along with event dimensions.")endtest "does not fail if using session metric with event:page dimension", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["bounce_rate"],"date_range" => "all","dimensions" => ["event:page"]@@ -788,6 +890,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "does not fail if using session metric with event filter", %{site: site} do%{+ "site_id" => site.domain,"metrics" => ["bounce_rate"],"date_range" => "all","filters" => [["is", "event:props:foo", ["(none)"]]]diff --git a/test/plausible/stats/query_result_test.exs b/test/plausible/stats/query_result_test.exsindex c4d279a69f..5e2ce5b4dd 100644--- a/test/plausible/stats/query_result_test.exs+++ b/test/plausible/stats/query_result_test.exs@@ -21,6 +21,7 @@ defmodule Plausible.Stats.QueryResultTest do{:ok, query} =Query.build(site,+ :public,%{"site_id" => site.domain,"metrics" => ["pageviews"],diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex 0c64082ac2..cbfda8e256 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -157,7 +157,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end- test "wildcard referrer filter with special regex characters", %{conn: conn, site: site} do+ test "contains referrer filter", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, referrer: "https://a.com"),build(:pageview, referrer: "https://a.com"),@@ -170,7 +170,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"date_range" => "all","metrics" => ["visitors"],"filters" => [- ["matches", "visit:referrer", ["**a.com**"]]+ ["contains", "visit:referrer", ["a.com"]]]})@@ -476,7 +476,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"date_range" => "all","metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],"filters" => [- ["matches", "event:hostname", ["*.example.com", "example.com"]]+ ["contains", "event:hostname", ["one.example.com", "example.com"]]]})@@ -732,7 +732,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end- test "wildcard page filter", %{conn: conn, site: site} do+ test "contains page filter", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, pathname: "/en/page1"),build(:pageview, pathname: "/en/page2"),@@ -745,14 +745,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"date_range" => "all","metrics" => ["visitors"],"filters" => [- ["matches", "event:page", ["/en/**"]]+ ["contains", "event:page", ["/en/"]]]})assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]end- test "negated wildcard page filter", %{conn: conn, site: site} do+ test "negated contains page filter", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, pathname: "/en/page1"),build(:pageview, pathname: "/en/page2"),@@ -765,14 +765,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"date_range" => "all","metrics" => ["visitors"],"filters" => [- ["does_not_match", "event:page", ["/en/**"]]+ ["does_not_contain", "event:page", ["/en/"]]]})assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]end- test "wildcard and member filter combined", %{conn: conn, site: site} do+ test "contains and member filter combined", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, pathname: "/en/page1"),build(:pageview, pathname: "/en/page2"),@@ -786,14 +786,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"date_range" => "all","metrics" => ["visitors"],"filters" => [- ["matches", "event:page", ["/en/**", "/pl/**"]]+ ["contains", "event:page", ["/en/", "/pl/"]]]})assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}]end- test "can escape pipe character in member + wildcard filter", %{conn: conn, site: site} do+ test "can escape pipe character in member + contains filter", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, pathname: "/blog/post|1"),build(:pageview, pathname: "/otherpost|1"),@@ -807,7 +807,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"date_range" => "all","metrics" => ["visitors"],"filters" => [- ["matches", "event:page", ["**post\\|1", "/something-else"]]+ ["contains", "event:page", ["post|1", "/something-else"]]]})@@ -1451,8 +1451,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"site_id" => site.domain,"date_range" => "all","metrics" => ["pageviews"],- "dimensions" => ["event:hostname"],- "with_imported" => "true"+ "dimensions" => ["event:hostname"]})%{"results" => results} = json_response(conn, 200)@@ -1667,7 +1666,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"date_range" => "all","dimensions" => ["event:name"],"filters" => [- ["matches", "event:page", ["/pageA"]]+ ["contains", "event:page", ["/pageA"]]]})@@ -1740,7 +1739,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"date_range" => "all","dimensions" => ["event:page"],"filters" => [- ["matches", "event:page", ["/en/**"]]+ ["contains", "event:page", ["/en/"]]]})@@ -1752,14 +1751,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end- test "can filter event:hostname with a wildcard", %{+ test "can filter event:hostname with a contains", %{conn: conn,site: site} dopopulate_stats(site, [- build(:pageview, hostname: "alice.example.com", pathname: "/a"),- build(:pageview, hostname: "anna.example.com", pathname: "/a"),- build(:pageview, hostname: "adam.example.com", pathname: "/a"),+ build(:pageview, hostname: "alice-m.example.com", pathname: "/a"),+ build(:pageview, hostname: "anna-m.example.com", pathname: "/a"),+ build(:pageview, hostname: "adam-m.example.com", pathname: "/a"),build(:pageview, hostname: "bob.example.com", pathname: "/b")])@@ -1770,7 +1769,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"date_range" => "all","dimensions" => ["event:page"],"filters" => [- ["matches", "event:hostname", ["a*.example.com"]]+ ["contains", "event:hostname", ["-m.example.com"]]]})diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exsindex d5bef31155..af5f322867 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs@@ -22,7 +22,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do})assert json_response(conn, 400)["error"] ==- "The owner of this site does not have access to the custom properties feature"+ "The owner of this site does not have access to the custom properties feature."endtest "can break down by an internal prop key without access to the props feature", %{@@ -64,7 +64,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do})assert json_response(conn, 400)["error"] ==- "The owner of this site does not have access to the custom properties feature"+ "The owner of this site does not have access to the custom properties feature."endtest "can filter by an internal prop key without access to the props feature", %{@@ -103,7 +103,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do})assert json_response(conn, 400)["error"] ==- "Metric `conversion_rate` can only be queried with event:goal filters or dimensions"+ "Metric `conversion_rate` can only be queried with event:goal filters or dimensions."endtest "validates that dimensions are valid", %{conn: conn, site: site} do@@ -115,7 +115,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do"dimensions" => ["badproperty"]})- assert json_response(conn, 400)["error"] =~ "Invalid dimensions"+ assert json_response(conn, 400)["error"] ==+ "#/dimensions/0: Invalid dimension \"badproperty\""endtest "empty custom property is invalid", %{conn: conn, site: site} do@@ -127,7 +128,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do"dimensions" => ["event:props:"]})- assert json_response(conn, 400)["error"] =~ "Invalid dimensions"+ assert json_response(conn, 400)["error"] ==+ "#/dimensions/0: Invalid dimension \"event:props:\""endtest "validates that correct date range is used", %{conn: conn, site: site} do@@ -139,7 +141,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do"dimensions" => ["event:name"]})- assert json_response(conn, 400)["error"] =~ "Invalid date_range"+ assert json_response(conn, 400)["error"] ==+ "#/date_range: Invalid date range \"bad_period\""endtest "fails when an invalid metric is provided", %{conn: conn, site: site} do@@ -151,7 +154,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do"dimensions" => ["event:name"]})- assert json_response(conn, 400)["error"] =~ "Unknown metric '\"baa\"'"+ assert json_response(conn, 400)["error"] == "#/metrics/1: Invalid metric \"baa\""endtest "session metrics cannot be used with event:name dimension", %{conn: conn, site: site} do@@ -164,7 +167,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do})assert json_response(conn, 400)["error"] =~- "Session metric(s) `bounce_rate` cannot be queried along with event dimensions"+ "Session metric(s) `bounce_rate` cannot be queried along with event dimensions."endtest "session metrics cannot be used with event:props:* dimension", %{conn: conn, site: site} do@@ -177,7 +180,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do})assert json_response(conn, 400)["error"] =~- "Session metric(s) `bounce_rate` cannot be queried along with event dimensions"+ "Session metric(s) `bounce_rate` cannot be queried along with event dimensions."endtest "validates that metric views_per_visit cannot be used with event:page filter", %{@@ -194,7 +197,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest doassert json_response(conn, 400) == %{"error" =>- "Metric `views_per_visit` cannot be queried with a filter on `event:page`"+ "Metric `views_per_visit` cannot be queried with a filter on `event:page`."}end@@ -211,7 +214,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do})assert json_response(conn, 400) == %{- "error" => "Metric `views_per_visit` cannot be queried with `dimensions`"+ "error" => "Metric `views_per_visit` cannot be queried with `dimensions`."}end@@ -227,7 +230,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do})assert json_response(conn, 400) == %{- "error" => "Metrics cannot be queried multiple times"+ "error" => "#/metrics: Expected items to be unique but they were not."}end@@ -252,7 +255,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest doassert json_response(conn, 400) == %{"error" =>- "Invalid visit:country filter, visit:country needs to be a valid 2-letter country code"+ "Invalid visit:country filter, visit:country needs to be a valid 2-letter country code."}endend