Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Debug queries for super-admins (#4010)

hq1dd493fdad21f
diff --git a/assets/js/dashboard/api.js b/assets/js/dashboard/api.js
index 26fd29916e..a14c938f96 100644
--- a/assets/js/dashboard/api.js
+++ b/assets/js/dashboard/api.js
@@ -61,9 +61,10 @@ export function serializeQuery(query, extraQuery = []) {
export function get(url, query = {}, ...extraQuery) {
const headers = SHARED_LINK_AUTH ? { 'X-Shared-Link-Auth': SHARED_LINK_AUTH } : {}
- url = url + serializeQuery(query, extraQuery)
- return fetch(url, { signal: abortController.signal, headers: headers })
+ const serializedUrl = url + serializeQuery(query, extraQuery)
+ return fetch(serializedUrl, { signal: abortController.signal, headers: headers })
.then(response => {
+ logDebugHeaders(url, response.headers)
if (!response.ok) {
return response.json().then((msg) => {
throw new ApiError(msg.error, msg)
@@ -73,6 +74,13 @@ export function get(url, query = {}, ...extraQuery) {
})
}
+function logDebugHeaders(url, headers) {
+ const debugHeaders = Array.from(headers).filter(([h]) => h.startsWith("x-plausible"))
+ if (debugHeaders.length > 0) {
+ console.info(url, Object.fromEntries(debugHeaders))
+ }
+}
+
export function put(url, body) {
return fetch(url, {
method: "PUT",
diff --git a/extra/lib/plausible/stats/funnel.ex b/extra/lib/plausible/stats/funnel.ex
index d9c497cee6..42248f5e18 100644
--- a/extra/lib/plausible/stats/funnel.ex
+++ b/extra/lib/plausible/stats/funnel.ex
@@ -74,7 +74,7 @@ defmodule Plausible.Stats.Funnel do
group_by: f.step
)
- ClickhouseRepo.all(query)
+ ClickhouseRepo.all(query, label: :query_funnel)
end
defp select_funnel(db_query, funnel_definition) do
diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex
index f6f8631d97..05b4184635 100644
--- a/lib/plausible/clickhouse_repo.ex
+++ b/lib/plausible/clickhouse_repo.ex
@@ -4,6 +4,8 @@ defmodule Plausible.ClickhouseRepo do
adapter: Ecto.Adapters.ClickHouse,
read_only: true
+ require Logger
+
defmacro __using__(_) do
quote do
alias Plausible.ClickhouseRepo
@@ -12,17 +14,69 @@ defmodule Plausible.ClickhouseRepo do
end
end
+ @impl Ecto.Repo
+ def prepare_query(operation, query, opts) do
+ if Plausible.DebugReplayInfo.super_admin?() do
+ Plausible.DebugReplayInfo.track_query(
+ to_inline_sql(operation, query),
+ opts[:label] || "unlabelled"
+ )
+ end
+
+ {query, include_log_comment(opts)}
+ end
+
@task_timeout 60_000
def parallel_tasks(queries) do
- ctx = OpenTelemetry.Ctx.get_current()
+ otel_ctx = OpenTelemetry.Ctx.get_current()
+ sentry_ctx = Sentry.Context.get_all()
execute_with_tracing = fn fun ->
- OpenTelemetry.Ctx.attach(ctx)
- fun.()
+ Plausible.DebugReplayInfo.carry_over_context(sentry_ctx)
+ OpenTelemetry.Ctx.attach(otel_ctx)
+ result = fun.()
+ {Sentry.Context.get_all(), result}
end
Task.async_stream(queries, execute_with_tracing, max_concurrency: 3, timeout: @task_timeout)
|> Enum.to_list()
|> Keyword.values()
+ |> Enum.map(fn {sentry_ctx, result} ->
+ set_sentry_context(sentry_ctx)
+ result
+ end)
+ end
+
+ defp set_sentry_context(previous_sentry_ctx) do
+ Plausible.DebugReplayInfo.carry_over_context(previous_sentry_ctx)
+ previous_queries = Plausible.DebugReplayInfo.get_queries_from_context(previous_sentry_ctx)
+ current_queries = Plausible.DebugReplayInfo.get_queries_from_context()
+
+ Sentry.Context.set_extra_context(%{
+ queries: previous_queries ++ current_queries
+ })
+ end
+
+ defp include_log_comment(opts) do
+ sentry_context = Sentry.Context.get_all()
+
+ log_comment = %{
+ user_id: sentry_context[:user][:id],
+ label: opts[:label] || "unlabelled",
+ url: sentry_context[:request][:url],
+ domain: sentry_context[:extra][:domain],
+ site_id: sentry_context[:extra][:site_id],
+ metadata: opts[:metadata] || %{}
+ }
+
+ case Jason.encode(log_comment) do
+ {:ok, encoded} ->
+ setting = {:log_comment, encoded}
+ Keyword.update(opts, :settings, [setting], fn settings -> [setting | settings] end)
+
+ {:error, _} ->
+ Logger.error("Failed to include log comment: #{inspect(log_comment)}")
+ opts
+ end
end
end
diff --git a/lib/plausible/debug_replay_info.ex b/lib/plausible/debug_replay_info.ex
index ed1b90fbf5..8d418f65bb 100644
--- a/lib/plausible/debug_replay_info.ex
+++ b/lib/plausible/debug_replay_info.ex
@@ -42,6 +42,40 @@ defmodule Plausible.DebugReplayInfo do
end
end
+ def super_admin? do
+ context = Sentry.Context.get_all()
+
+ case context[:user] do
+ %{super_admin?: true} ->
+ true
+
+ _ ->
+ false
+ end
+ end
+
+ def track_query(query, label) do
+ queries = get_queries_from_context()
+
+ Sentry.Context.set_extra_context(%{
+ queries: [%{label => query} | queries]
+ })
+ end
+
+ def carry_over_context(sentry_ctx) do
+ Sentry.Context.set_user_context(sentry_ctx[:user])
+ Sentry.Context.set_request_context(sentry_ctx[:request])
+
+ Sentry.Context.set_extra_context(%{
+ domain: sentry_ctx[:extra][:domain],
+ site_id: sentry_ctx[:extra][:site_id]
+ })
+ end
+
+ def get_queries_from_context(context \\ Sentry.Context.get_all()) do
+ get_in(context, [:extra, :queries]) || []
+ end
+
@spec deserialize(String.t()) :: any()
def deserialize(replay_info) do
replay_info
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
index 27d4290b01..dcf8d3b88e 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/aggregate.ex
@@ -46,7 +46,7 @@ defmodule Plausible.Stats.Aggregate do
from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))
|> merge_imported(site, query, :aggregate, metrics)
|> maybe_add_conversion_rate(site, query, metrics, include_imported: query.include_imported)
- |> ClickhouseRepo.one()
+ |> ClickhouseRepo.one(label: :aggregate_events)
end
defp aggregate_sessions(_, _, []), do: %{}
@@ -55,7 +55,7 @@ defmodule Plausible.Stats.Aggregate do
from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))
|> filter_converted_sessions(site, query)
|> merge_imported(site, query, :aggregate, metrics)
- |> ClickhouseRepo.one()
+ |> ClickhouseRepo.one(label: :aggregate_sessions)
|> Util.keep_requested_metrics(metrics)
end
@@ -187,7 +187,7 @@ defmodule Plausible.Stats.Aggregate do
from e in Ecto.Query.subquery(avg_time_per_page_transition_q),
select: fragment("avg(ifNotFinite(?,NULL))", e.avg)
- %{time_on_page: ClickhouseRepo.one(time_on_page_q)}
+ %{time_on_page: ClickhouseRepo.one(time_on_page_q, label: :time_on_page)}
end
@metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent]
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index e2b0b40667..e0bb1c871b 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -111,7 +111,7 @@ defmodule Plausible.Stats.Breakdown do
if full_q do
full_q
|> maybe_add_conversion_rate(site, query, metrics, include_imported: false)
- |> ClickhouseRepo.all()
+ |> ClickhouseRepo.all(label: :breakdown_by_goal)
|> transform_keys(%{name: :goal})
|> cast_revenue_metrics_to_money(revenue_goals)
|> Util.keep_requested_metrics(metrics)
@@ -309,7 +309,7 @@ defmodule Plausible.Stats.Breakdown do
defp paginate_and_execute(q, metrics, pagination) do
q
|> apply_pagination(pagination)
- |> ClickhouseRepo.all()
+ |> ClickhouseRepo.all(label: :paginate_and_execute)
|> Util.keep_requested_metrics(metrics)
end
@@ -405,7 +405,7 @@ defmodule Plausible.Stats.Breakdown do
end
timed_pages_q
- |> Plausible.ClickhouseRepo.all()
+ |> Plausible.ClickhouseRepo.all(label: :breakdown_time_on_page)
|> Map.new()
end
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex
index 52d2e889ce..6c90bcf0ad 100644
--- a/lib/plausible/stats/clickhouse.ex
+++ b/lib/plausible/stats/clickhouse.ex
@@ -46,7 +46,7 @@ defmodule Plausible.Stats.Clickhouse do
group_by: i.import_id,
select: {i.import_id, sum(i.pageviews)}
)
- |> Plausible.ClickhouseRepo.all()
+ |> Plausible.ClickhouseRepo.all(label: :imported_pageview_counts)
|> Map.new()
end
@@ -117,7 +117,7 @@ defmodule Plausible.Stats.Clickhouse do
referrers = Plausible.Stats.Sampling.add_query_hint(referrers, 10_000_000)
end
- ClickhouseRepo.all(referrers)
+ ClickhouseRepo.all(referrers, label: :referrers)
end
def current_visitors(site, query) do
@@ -168,7 +168,7 @@ defmodule Plausible.Stats.Clickhouse do
previous_result =
previous_query
- |> ClickhouseRepo.all()
+ |> ClickhouseRepo.all(label: :last_24h_visitors_previous)
|> Enum.reduce(%{}, fn
%{total_visitors: total, site_id: site_id}, acc -> Map.put_new(acc, site_id, total)
end)
@@ -199,7 +199,7 @@ defmodule Plausible.Stats.Clickhouse do
result =
current_q
- |> ClickhouseRepo.all()
+ |> ClickhouseRepo.all(label: :last_24h_visitors_current)
|> Enum.group_by(& &1.site_id)
|> Enum.map(fn {site_id, entries} ->
%{total: visitors} = List.first(entries)
diff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.ex
index 8fe69b9d44..c2d8ad7716 100644
--- a/lib/plausible/stats/custom_props.ex
+++ b/lib/plausible/stats/custom_props.ex
@@ -19,7 +19,7 @@ defmodule Plausible.Stats.CustomProps do
distinct: true
)
|> maybe_allowed_props_only(site)
- |> ClickhouseRepo.all()
+ |> ClickhouseRepo.all(label: :fetch_prop_names)
end
end
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index 3b400907b1..6908e2761c 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -58,7 +58,7 @@ defmodule Plausible.Stats.Timeseries do
|> select_bucket(site, query)
|> maybe_add_timeseries_conversion_rate(site, query, metrics)
|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)
- |> ClickhouseRepo.all()
+ |> ClickhouseRepo.all(label: :event_timeseries)
end
defp sessions_timeseries(_, _, []), do: []
@@ -68,7 +68,7 @@ defmodule Plausible.Stats.Timeseries do
|> filter_converted_sessions(site, query)
|> select_bucket(site, query)
|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)
- |> ClickhouseRepo.all()
+ |> ClickhouseRepo.all(label: :session_timeseries)
|> Util.keep_requested_metrics(metrics)
end
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 8b411eef0b..f85765941e 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -12,7 +12,8 @@ defmodule PlausibleWeb.Api.StatsController do
@revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
- plug(:date_validation_plug)
+ plug :date_validation_plug
+ plug PlausibleWeb.Plugs.InjectDebugHeaders
@doc """
Returns a time-series based on given parameters.
diff --git a/lib/plausible_web/plugs/authorize_site_access.ex b/lib/plausible_web/plugs/authorize_site_access.ex
index de35461c50..3306a5229e 100644
--- a/lib/plausible_web/plugs/authorize_site_access.ex
+++ b/lib/plausible_web/plugs/authorize_site_access.ex
@@ -20,14 +20,17 @@ defmodule PlausibleWeb.AuthorizeSiteAccess do
PlausibleWeb.ControllerHelpers.render_error(conn, 404) |> halt
else
user_id = get_session(conn, :current_user_id)
+
membership_role = user_id && Plausible.Sites.role(user_id, site)
+ super_admin? = user_id && Plausible.Auth.is_super_admin?(user_id)
+
role =
cond do
user_id && membership_role ->
membership_role
- Plausible.Auth.is_super_admin?(user_id) ->
+ super_admin? ->
:super_admin
site.public ->
@@ -41,7 +44,8 @@ defmodule PlausibleWeb.AuthorizeSiteAccess do
end
if role in allowed_roles do
- Sentry.Context.set_user_context(%{id: user_id})
+ Sentry.Context.set_user_context(%{id: user_id, super_admin?: super_admin?})
+
Plausible.OpenTelemetry.add_user_attributes(user_id)
Sentry.Context.set_extra_context(%{site_id: site.id, domain: site.domain})
@@ -49,7 +53,7 @@ defmodule PlausibleWeb.AuthorizeSiteAccess do
site = Plausible.Imported.load_import_data(site)
- merge_assigns(conn, site: site, current_user_role: role)
+ merge_assigns(conn, site: site, current_user_role: role, current_user_id: user_id)
else
PlausibleWeb.ControllerHelpers.render_error(conn, 404) |> halt
end
diff --git a/lib/plausible_web/plugs/inject_debug_headers.ex b/lib/plausible_web/plugs/inject_debug_headers.ex
new file mode 100644
index 0000000000..b0f3cf77ba
--- /dev/null
+++ b/lib/plausible_web/plugs/inject_debug_headers.ex
@@ -0,0 +1,25 @@
+defmodule PlausibleWeb.Plugs.InjectDebugHeaders do
+ @moduledoc """
+ This plug updates the response with debug query headers,
+ granted they were tracked via `Plausible.DebugReplayInfo`.
+ """
+
+ def init(opts), do: opts
+
+ def call(conn, _opts \\ []) do
+ Plug.Conn.register_before_send(conn, fn conn ->
+ Plausible.DebugReplayInfo.get_queries_from_context()
+ |> Enum.reverse()
+ |> Enum.with_index()
+ |> Enum.reduce(conn, fn {q, index}, conn ->
+ {[label], [value]} = Enum.unzip(q)
+
+ conn
+ |> Plug.Conn.put_resp_header(
+ "x-plausible-query-#{String.pad_leading("#{index}", 3, "0")}-#{label}",
+ String.replace(value, ["\n", "\r", "\x00"], "")
+ )
+ end)
+ end)
+ end
+end
diff --git a/mix.exs b/mix.exs
index 2166808efa..f839133bf4 100644
--- a/mix.exs
+++ b/mix.exs
@@ -69,7 +69,7 @@ defmodule Plausible.MixProject do
{:bamboo_mua, "~> 0.1.4"},
{:bcrypt_elixir, "~> 3.0"},
{:bypass, "~> 2.1", only: [:dev, :test, :ce_test]},
- {:ecto_ch, "~> 0.3"},
+ {:ecto_ch, github: "plausible/ecto_ch", branch: "pretty-sql"},
{:cloak, "~> 1.1"},
{:cloak_ecto, "~> 1.2"},
{:combination, "~> 0.0.3"},
diff --git a/mix.lock b/mix.lock
index 240acdbc55..5918ee3859 100644
--- a/mix.lock
+++ b/mix.lock
@@ -8,7 +8,7 @@
"bcrypt_elixir": {:hex, :bcrypt_elixir, "3.1.0", "0b110a9a6c619b19a7f73fa3004aa11d6e719a67e672d1633dc36b6b2290a0f7", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "2ad2acb5a8bc049e8d5aa267802631912bb80d5f4110a178ae7999e69dca1bf7"},
"bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
"bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"},
- "castore": {:hex, :castore, "1.0.5", "9eeebb394cc9a0f3ae56b813459f990abb0a3dedee1be6b27fdb50301930502f", [:mix], [], "hexpm", "8d7c597c3e4a64c395980882d4bca3cebb8d74197c590dc272cfd3b6a6310578"},
+ "castore": {:hex, :castore, "1.0.6", "ffc42f110ebfdafab0ea159cd43d31365fa0af0ce4a02ecebf1707ae619ee727", [:mix], [], "hexpm", "374c6e7ca752296be3d6780a6d5b922854ffcc74123da90f2f328996b962d33a"},
"certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"},
"ch": {:hex, :ch, "0.2.5", "b8d70689951bd14c8c8791dc72cdc957ba489ceae723e79cf1a91d95b6b855ae", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "97de104c8f513a23c6d673da37741f68ae743f6cdb654b96a728d382e2fba4de"},
"chatterbox": {:hex, :ts_chatterbox, "0.15.1", "5cac4d15dd7ad61fc3c4415ce4826fc563d4643dee897a558ec4ea0b1c835c9c", [:rebar3], [{:hpack, "~> 0.3.0", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "4f75b91451338bc0da5f52f3480fa6ef6e3a2aeecfc33686d6b3d0a0948f31aa"},
@@ -32,8 +32,8 @@
"digital_token": {:hex, :digital_token, "0.6.0", "13e6de581f0b1f6c686f7c7d12ab11a84a7b22fa79adeb4b50eec1a2d278d258", [:mix], [{:cldr_utils, "~> 2.17", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "2455d626e7c61a128b02a4a8caddb092548c3eb613ac6f6a85e4cbb6caddc4d1"},
"double": {:hex, :double, "0.8.2", "8e1cfcccdaef76c18846bc08e555555a2a699b806fa207b6468572a60513cc6a", [:mix], [], "hexpm", "90287642b2ec86125e0457aaba2ab0e80f7d7050cc80a0cef733e59bd70aa67c"},
"earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"},
- "ecto": {:hex, :ecto, "3.11.1", "4b4972b717e7ca83d30121b12998f5fcdc62ba0ed4f20fd390f16f3270d85c3e", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebd3d3772cd0dfcd8d772659e41ed527c28b2a8bde4b00fe03e0463da0f1983b"},
- "ecto_ch": {:hex, :ecto_ch, "0.3.2", "b6e7d0a6ad412662d7727ba1b5128a1a30a0835e1b4168db79e5b2551a27ba50", [:mix], [{:ch, "~> 0.2.0", [hex: :ch, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.9", [hex: :ecto_sql, repo: "hexpm", optional: false]}], "hexpm", "a195d0f09d1177d9b11f00ef608895db1e02711d0a18730e98fbb50055758e6d"},
+ "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"},
+ "ecto_ch": {:git, "https://github.com/plausible/ecto_ch.git", "6a3a74364f514810c4f073575a33ab3be891f560", [branch: "pretty-sql"]},
"ecto_network": {:hex, :ecto_network, "1.5.0", "a930c910975e7a91237b858ebf0f4ad7b2aae32fa846275aa203cb858459ec73", [:mix], [{:ecto_sql, ">= 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:phoenix_html, ">= 0.0.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.14.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "4d614434ae3e6d373a2f693d56aafaa3f3349714668ffd6d24e760caf578aa2f"},
"ecto_sql": {:hex, :ecto_sql, "3.11.1", "e9abf28ae27ef3916b43545f9578b4750956ccea444853606472089e7d169470", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ce14063ab3514424276e7e360108ad6c2308f6d88164a076aac8a387e1fea634"},
"elixir_make": {:hex, :elixir_make, "0.7.7", "7128c60c2476019ed978210c245badf08b03dbec4f24d05790ef791da11aa17c", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5bc19fff950fad52bbe5f211b12db9ec82c6b34a9647da0c2224b8b8464c7e6c"},
@@ -121,7 +121,7 @@
"plug": {:hex, :plug, "1.15.3", "712976f504418f6dff0a3e554c40d705a9bcf89a7ccef92fc6a5ef8f16a30a97", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4365a3c010a56af402e0809208873d113e9c38c401cabd88027ef4f5c01fd2"},
"plug_cowboy": {:hex, :plug_cowboy, "2.7.0", "3ae9369c60641084363b08fe90267cbdd316df57e3557ea522114b30b63256ea", [:mix], [{:cowboy, "~> 2.7.0 or ~> 2.8.0 or ~> 2.9.0 or ~> 2.10.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d85444fb8aa1f2fc62eabe83bbe387d81510d773886774ebdcb429b3da3c1a4a"},
"plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"},
- "postgrex": {:hex, :postgrex, "0.17.4", "5777781f80f53b7c431a001c8dad83ee167bcebcf3a793e3906efff680ab62b3", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "6458f7d5b70652bc81c3ea759f91736c16a31be000f306d3c64bcdfe9a18b3cc"},
+ "postgrex": {:hex, :postgrex, "0.17.5", "0483d054938a8dc069b21bdd636bf56c487404c241ce6c319c1f43588246b281", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "50b8b11afbb2c4095a3ba675b4f055c416d0f3d7de6633a595fc131a828a67eb"},
"prom_ex": {:hex, :prom_ex, "1.9.0", "63e6dda6c05cdeec1f26c48443dcc38ffd2118b3665ae8d2bd0e5b79f2aea03e", [:mix], [{:absinthe, ">= 1.6.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.0.2", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.5.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.15", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.4.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.3", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.5.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.14.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.12.1", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.5", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "01f3d4f69ec93068219e686cc65e58a29c42bea5429a8ff4e2121f19db178ee6"},
"public_suffix": {:git, "https://github.com/axelson/publicsuffix-elixir", "fa40c243d4b5d8598b90cff268bc4e33f3bb63f1", []},
"ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
diff --git a/test/plausible/clickhouse_repo_test.exs b/test/plausible/clickhouse_repo_test.exs
new file mode 100644
index 0000000000..841bcd1365
--- /dev/null
+++ b/test/plausible/clickhouse_repo_test.exs
@@ -0,0 +1,110 @@
+defmodule Plausible.ClickhouseRepoTest do
+ use Plausible.DataCase, async: true
+
+ alias Plausible.ClickhouseRepo
+ alias Plausible.DebugReplayInfo
+
+ import ExUnit.CaptureLog
+
+ test "queries are kept in context for debugging purposes for superadmins" do
+ Sentry.Context.set_user_context(%{id: 1, super_admin?: true})
+
+ ClickhouseRepo.all(from(u in "events_v2", select: true, limit: 0))
+
+ ClickhouseRepo.one(from(u in "events_v2", select: true, limit: 0),
+ label: "one"
+ )
+
+ queries = DebugReplayInfo.get_queries_from_context()
+
+ assert [
+ %{"one" => "SELECT true FROM \"events_v2\" AS e0 LIMIT 0"},
+ %{"unlabelled" => "SELECT true FROM \"events_v2\" AS e0 LIMIT 0"}
+ ] = queries
+ end
+
+ test "queries are not kept in context for debugging purposes for non-superadmins" do
+ ClickhouseRepo.all(from(u in "events_v2", select: true, limit: 0))
+
+ ClickhouseRepo.one(from(u in "events_v2", select: true, limit: 0),
+ label: "one"
+ )
+
+ assert DebugReplayInfo.get_queries_from_context() == []
+ end
+
+ test "queries are logged with sentry and extra metadata" do
+ Sentry.Context.set_user_context(%{id: 1})
+ Sentry.Context.set_request_context(%{url: "http://example.com"})
+ Sentry.Context.set_extra_context(%{domain: "example.com", site_id: 1})
+
+ ClickhouseRepo.all(from(u in "sessions_v2", select: true, limit: 0),
+ label: "log_all"
+ )
+
+ ClickhouseRepo.one(from(u in "sessions_v2", select: true, limit: 0),
+ label: "log_one",
+ metadata: %{"metric" => "value"}
+ )
+
+ flush_clickhouse_logs()
+
+ assert %{
+ rows: [
+ [q1, c1],
+ [q2, c2]
+ ]
+ } =
+ ClickhouseRepo.query!("""
+ SELECT
+ query,
+ log_comment
+ FROM system.query_log
+ WHERE (type = 1) AND (query LIKE '%sessions_v2%')
+ AND JSONExtractString(log_comment, 'label') IN ('log_all', 'log_one')
+ ORDER BY event_time DESC
+ LIMIT 2
+ """)
+
+ assert q1 == "SELECT true FROM \"sessions_v2\" AS s0 LIMIT 0"
+ assert q1 == q2
+
+ assert Enum.find([c1, c2], fn c ->
+ Jason.decode!(c) == %{
+ "label" => "log_all",
+ "domain" => "example.com",
+ "url" => "http://example.com",
+ "user_id" => 1,
+ "metadata" => %{},
+ "site_id" => 1
+ }
+ end)
+
+ assert Enum.find([c1, c2], fn c ->
+ Jason.decode!(c) == %{
+ "label" => "log_one",
+ "domain" => "example.com",
+ "url" => "http://example.com",
+ "user_id" => 1,
+ "metadata" => %{"metric" => "value"},
+ "site_id" => 1
+ }
+ end)
+ end
+
+ test "non-serializable log comment won't cause the query to crash and will log an error" do
+ log =
+ capture_log(fn ->
+ assert [] =
+ ClickhouseRepo.all(from(u in "sessions_v2", select: true, limit: 0),
+ label: "log_all",
+ metadata: {:error, :skip_me}
+ )
+ end)
+
+ assert log =~
+ ~s/Failed to include log comment: %{label: "log_all", domain: nil, metadata: {:error, :skip_me}, url: nil, site_id: nil, user_id: nil}/
+ end
+
+ defp flush_clickhouse_logs(), do: Plausible.IngestRepo.query!("SYSTEM FLUSH LOGS")
+end
diff --git a/test/plausible/debug_replay_info_test.exs b/test/plausible/debug_replay_info_test.exs
index b0379aa155..89c6ce6cab 100644
--- a/test/plausible/debug_replay_info_test.exs
+++ b/test/plausible/debug_replay_info_test.exs
@@ -46,4 +46,31 @@ defmodule Plausible.DebugReplayInfoTest do
assert context.extra.debug_replay_info == :too_large
assert context.extra.debug_replay_info_size > 10_000
end
+
+ describe "query tracking" do
+ test "track and retrieve queries" do
+ :ok = Plausible.DebugReplayInfo.track_query("SELECT * FROM users", "users")
+ :ok = Plausible.DebugReplayInfo.track_query("SELECT * FROM accounts", "accounts")
+
+ assert [%{"accounts" => "SELECT * FROM accounts"}, %{"users" => "SELECT * FROM users"}] =
+ Plausible.DebugReplayInfo.get_queries_from_context()
+ end
+
+ test "carry over context" do
+ Sentry.Context.set_user_context(%{id: 1})
+ Sentry.Context.set_request_context(%{url: "http://example.com"})
+ Sentry.Context.set_extra_context(%{domain: "example.com", site_id: 1})
+
+ test = self()
+
+ sentry_ctx = Sentry.Context.get_all()
+
+ Task.start(fn ->
+ Plausible.DebugReplayInfo.carry_over_context(sentry_ctx)
+ send(test, {:task_context, Sentry.Context.get_all()})
+ end)
+
+ assert_receive {:task_context, ^sentry_ctx}
+ end
+ end
end
diff --git a/test/plausible_web/plugs/inject_debug_headers_test.exs b/test/plausible_web/plugs/inject_debug_headers_test.exs
new file mode 100644
index 0000000000..b8db4556f4
--- /dev/null
+++ b/test/plausible_web/plugs/inject_debug_headers_test.exs
@@ -0,0 +1,39 @@
+defmodule PlausibleWeb.Plugs.InjectDebugHeadersTest do
+ use Plausible.DataCase, async: true
+ use Plug.Test
+
+ alias PlausibleWeb.Plugs.InjectDebugHeaders
+ @prefix "x-plausible-"
+
+ test "does not inject #{@prefix} headers when no queries are registered" do
+ conn = :get |> conn("/") |> InjectDebugHeaders.call() |> send_resp(200, "")
+ assert Enum.filter(conn.resp_headers, fn {k, _} -> String.starts_with?(k, @prefix) end) == []
+ end
+
+ test "injects #{@prefix} headers when queries are registered" do
+ :ok = Plausible.DebugReplayInfo.track_query("SELECT * FROM users", "users")
+ :ok = Plausible.DebugReplayInfo.track_query("SELECT * FROM accounts", "accounts")
+
+ conn = :get |> conn("/") |> InjectDebugHeaders.call() |> send_resp(200, "")
+
+ assert Enum.filter(conn.resp_headers, fn {k, _} -> String.starts_with?(k, @prefix) end) ==
+ [
+ {"x-plausible-query-000-users", "SELECT * FROM users"},
+ {"x-plausible-query-001-accounts", "SELECT * FROM accounts"}
+ ]
+ end
+
+ test "skips invalid header chars" do
+ :ok = Plausible.DebugReplayInfo.track_query("\nfoo", "trap1")
+ :ok = Plausible.DebugReplayInfo.track_query("\rbar", "trap2")
+ :ok = Plausible.DebugReplayInfo.track_query("\x00baz", "trap3")
+ conn = :get |> conn("/") |> InjectDebugHeaders.call() |> send_resp(200, "")
+
+ assert Enum.filter(conn.resp_headers, fn {k, _} -> String.starts_with?(k, @prefix) end) ==
+ [
+ {"x-plausible-query-000-trap1", "foo"},
+ {"x-plausible-query-001-trap2", "bar"},
+ {"x-plausible-query-002-trap3", "baz"}
+ ]
+ end
+end