Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Revert "Debug queries for super-admins (#4010)" (#4073)
Adrian Gruntkowskid7ca8d96005b
diff --git a/assets/js/dashboard/api.js b/assets/js/dashboard/api.jsindex a14c938f96..26fd29916e 100644--- a/assets/js/dashboard/api.js+++ b/assets/js/dashboard/api.js@@ -61,10 +61,9 @@ export function serializeQuery(query, extraQuery = []) {export function get(url, query = {}, ...extraQuery) {const headers = SHARED_LINK_AUTH ? { 'X-Shared-Link-Auth': SHARED_LINK_AUTH } : {}- const serializedUrl = url + serializeQuery(query, extraQuery)- return fetch(serializedUrl, { signal: abortController.signal, headers: headers })+ url = url + serializeQuery(query, extraQuery)+ return fetch(url, { signal: abortController.signal, headers: headers }).then(response => {- logDebugHeaders(url, response.headers)if (!response.ok) {return response.json().then((msg) => {throw new ApiError(msg.error, msg)@@ -74,13 +73,6 @@ export function get(url, query = {}, ...extraQuery) {})}-function logDebugHeaders(url, headers) {- const debugHeaders = Array.from(headers).filter(([h]) => h.startsWith("x-plausible"))- if (debugHeaders.length > 0) {- console.info(url, Object.fromEntries(debugHeaders))- }-}-export function put(url, body) {return fetch(url, {method: "PUT",diff --git a/extra/lib/plausible/stats/funnel.ex b/extra/lib/plausible/stats/funnel.exindex 42248f5e18..d9c497cee6 100644--- a/extra/lib/plausible/stats/funnel.ex+++ b/extra/lib/plausible/stats/funnel.ex@@ -74,7 +74,7 @@ defmodule Plausible.Stats.Funnel dogroup_by: f.step)- ClickhouseRepo.all(query, label: :query_funnel)+ ClickhouseRepo.all(query)enddefp select_funnel(db_query, funnel_definition) dodiff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.exindex 05b4184635..f6f8631d97 100644--- a/lib/plausible/clickhouse_repo.ex+++ b/lib/plausible/clickhouse_repo.ex@@ -4,8 +4,6 @@ defmodule Plausible.ClickhouseRepo doadapter: Ecto.Adapters.ClickHouse,read_only: true- require Logger-defmacro __using__(_) doquote doalias Plausible.ClickhouseRepo@@ -14,69 +12,17 @@ defmodule Plausible.ClickhouseRepo doendend- @impl Ecto.Repo- def prepare_query(operation, query, opts) do- if Plausible.DebugReplayInfo.super_admin?() do- Plausible.DebugReplayInfo.track_query(- to_inline_sql(operation, query),- opts[:label] || "unlabelled"- )- end-- {query, include_log_comment(opts)}- end-@task_timeout 60_000def parallel_tasks(queries) do- otel_ctx = OpenTelemetry.Ctx.get_current()- sentry_ctx = Sentry.Context.get_all()+ ctx = OpenTelemetry.Ctx.get_current()execute_with_tracing = fn fun ->- Plausible.DebugReplayInfo.carry_over_context(sentry_ctx)- OpenTelemetry.Ctx.attach(otel_ctx)- result = fun.()- {Sentry.Context.get_all(), result}+ OpenTelemetry.Ctx.attach(ctx)+ fun.()endTask.async_stream(queries, execute_with_tracing, max_concurrency: 3, timeout: @task_timeout)|> Enum.to_list()|> Keyword.values()- |> Enum.map(fn {sentry_ctx, result} ->- set_sentry_context(sentry_ctx)- result- end)- end-- defp set_sentry_context(previous_sentry_ctx) do- Plausible.DebugReplayInfo.carry_over_context(previous_sentry_ctx)- previous_queries = Plausible.DebugReplayInfo.get_queries_from_context(previous_sentry_ctx)- current_queries = Plausible.DebugReplayInfo.get_queries_from_context()-- Sentry.Context.set_extra_context(%{- queries: previous_queries ++ current_queries- })- end-- defp include_log_comment(opts) do- sentry_context = Sentry.Context.get_all()-- log_comment = %{- user_id: sentry_context[:user][:id],- label: opts[:label] || "unlabelled",- url: sentry_context[:request][:url],- domain: sentry_context[:extra][:domain],- site_id: sentry_context[:extra][:site_id],- metadata: opts[:metadata] || %{}- }-- case Jason.encode(log_comment) do- {:ok, encoded} ->- setting = {:log_comment, encoded}- Keyword.update(opts, :settings, [setting], fn settings -> [setting | settings] end)-- {:error, _} ->- Logger.error("Failed to include log comment: #{inspect(log_comment)}")- opts- endendenddiff --git a/lib/plausible/debug_replay_info.ex b/lib/plausible/debug_replay_info.exindex 8d418f65bb..ed1b90fbf5 100644--- a/lib/plausible/debug_replay_info.ex+++ b/lib/plausible/debug_replay_info.ex@@ -42,40 +42,6 @@ defmodule Plausible.DebugReplayInfo doendend- def super_admin? do- context = Sentry.Context.get_all()-- case context[:user] do- %{super_admin?: true} ->- true-- _ ->- false- end- end-- def track_query(query, label) do- queries = get_queries_from_context()-- Sentry.Context.set_extra_context(%{- queries: [%{label => query} | queries]- })- end-- def carry_over_context(sentry_ctx) do- Sentry.Context.set_user_context(sentry_ctx[:user])- Sentry.Context.set_request_context(sentry_ctx[:request])-- Sentry.Context.set_extra_context(%{- domain: sentry_ctx[:extra][:domain],- site_id: sentry_ctx[:extra][:site_id]- })- end-- def get_queries_from_context(context \\ Sentry.Context.get_all()) do- get_in(context, [:extra, :queries]) || []- end-@spec deserialize(String.t()) :: any()def deserialize(replay_info) doreplay_infodiff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex a9335a39fd..e1055e420f 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -46,7 +46,7 @@ defmodule Plausible.Stats.Aggregate dofrom(e in base_event_query(site, query), select: ^select_event_metrics(metrics))|> merge_imported(site, query, metrics)|> maybe_add_conversion_rate(site, query, metrics, include_imported: query.include_imported)- |> ClickhouseRepo.one(label: :aggregate_events)+ |> ClickhouseRepo.one()enddefp aggregate_sessions(_, _, []), do: %{}@@ -55,7 +55,7 @@ defmodule Plausible.Stats.Aggregate dofrom(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))|> filter_converted_sessions(site, query)|> merge_imported(site, query, metrics)- |> ClickhouseRepo.one(label: :aggregate_sessions)+ |> ClickhouseRepo.one()|> Util.keep_requested_metrics(metrics)end@@ -187,7 +187,7 @@ defmodule Plausible.Stats.Aggregate dofrom e in Ecto.Query.subquery(avg_time_per_page_transition_q),select: fragment("avg(ifNotFinite(?,NULL))", e.avg)- %{time_on_page: ClickhouseRepo.one(time_on_page_q, label: :time_on_page)}+ %{time_on_page: ClickhouseRepo.one(time_on_page_q)}end@metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent]diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 0efcedc8dc..7512df1a3f 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -121,7 +121,7 @@ defmodule Plausible.Stats.Breakdown doif full_q dofull_q|> maybe_add_conversion_rate(site, query, metrics, include_imported: false)- |> ClickhouseRepo.all(label: :breakdown_by_goal)+ |> ClickhouseRepo.all()|> transform_keys(%{name: :goal})|> cast_revenue_metrics_to_money(revenue_goals)|> Util.keep_requested_metrics(metrics)@@ -327,7 +327,7 @@ defmodule Plausible.Stats.Breakdown dodefp paginate_and_execute(q, metrics, pagination) doq|> apply_pagination(pagination)- |> ClickhouseRepo.all(label: :paginate_and_execute)+ |> ClickhouseRepo.all()|> Util.keep_requested_metrics(metrics)end@@ -423,7 +423,7 @@ defmodule Plausible.Stats.Breakdown doendtimed_pages_q- |> Plausible.ClickhouseRepo.all(label: :breakdown_time_on_page)+ |> Plausible.ClickhouseRepo.all()|> Map.new()enddiff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex 6c90bcf0ad..52d2e889ce 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -46,7 +46,7 @@ defmodule Plausible.Stats.Clickhouse dogroup_by: i.import_id,select: {i.import_id, sum(i.pageviews)})- |> Plausible.ClickhouseRepo.all(label: :imported_pageview_counts)+ |> Plausible.ClickhouseRepo.all()|> Map.new()end@@ -117,7 +117,7 @@ defmodule Plausible.Stats.Clickhouse doreferrers = Plausible.Stats.Sampling.add_query_hint(referrers, 10_000_000)end- ClickhouseRepo.all(referrers, label: :referrers)+ ClickhouseRepo.all(referrers)enddef current_visitors(site, query) do@@ -168,7 +168,7 @@ defmodule Plausible.Stats.Clickhouse doprevious_result =previous_query- |> ClickhouseRepo.all(label: :last_24h_visitors_previous)+ |> ClickhouseRepo.all()|> Enum.reduce(%{}, fn%{total_visitors: total, site_id: site_id}, acc -> Map.put_new(acc, site_id, total)end)@@ -199,7 +199,7 @@ defmodule Plausible.Stats.Clickhouse doresult =current_q- |> ClickhouseRepo.all(label: :last_24h_visitors_current)+ |> ClickhouseRepo.all()|> Enum.group_by(& &1.site_id)|> Enum.map(fn {site_id, entries} ->%{total: visitors} = List.first(entries)diff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.exindex c2d8ad7716..8fe69b9d44 100644--- a/lib/plausible/stats/custom_props.ex+++ b/lib/plausible/stats/custom_props.ex@@ -19,7 +19,7 @@ defmodule Plausible.Stats.CustomProps dodistinct: true)|> maybe_allowed_props_only(site)- |> ClickhouseRepo.all(label: :fetch_prop_names)+ |> ClickhouseRepo.all()endenddiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex baafd0e279..0ca145330b 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -58,7 +58,7 @@ defmodule Plausible.Stats.Timeseries do|> select_bucket(:events, site, query)|> maybe_add_timeseries_conversion_rate(site, query, metrics)|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)- |> ClickhouseRepo.all(label: :event_timeseries)+ |> ClickhouseRepo.all()enddefp sessions_timeseries(_, _, []), do: []@@ -68,7 +68,7 @@ defmodule Plausible.Stats.Timeseries do|> filter_converted_sessions(site, query)|> select_bucket(:sessions, site, query)|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)- |> ClickhouseRepo.all(label: :session_timeseries)+ |> ClickhouseRepo.all()|> Util.keep_requested_metrics(metrics)enddiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex da0a55b5e8..a36b3f6791 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -12,8 +12,7 @@ defmodule PlausibleWeb.Api.StatsController do@revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])- plug :date_validation_plug- plug PlausibleWeb.Plugs.InjectDebugHeaders+ plug(:date_validation_plug)@doc """Returns a time-series based on given parameters.diff --git a/lib/plausible_web/plugs/authorize_site_access.ex b/lib/plausible_web/plugs/authorize_site_access.exindex 3306a5229e..de35461c50 100644--- a/lib/plausible_web/plugs/authorize_site_access.ex+++ b/lib/plausible_web/plugs/authorize_site_access.ex@@ -20,17 +20,14 @@ defmodule PlausibleWeb.AuthorizeSiteAccess doPlausibleWeb.ControllerHelpers.render_error(conn, 404) |> haltelseuser_id = get_session(conn, :current_user_id)-membership_role = user_id && Plausible.Sites.role(user_id, site)- super_admin? = user_id && Plausible.Auth.is_super_admin?(user_id)-role =cond douser_id && membership_role ->membership_role- super_admin? ->+ Plausible.Auth.is_super_admin?(user_id) ->:super_adminsite.public ->@@ -44,8 +41,7 @@ defmodule PlausibleWeb.AuthorizeSiteAccess doendif role in allowed_roles do- Sentry.Context.set_user_context(%{id: user_id, super_admin?: super_admin?})-+ Sentry.Context.set_user_context(%{id: user_id})Plausible.OpenTelemetry.add_user_attributes(user_id)Sentry.Context.set_extra_context(%{site_id: site.id, domain: site.domain})@@ -53,7 +49,7 @@ defmodule PlausibleWeb.AuthorizeSiteAccess dosite = Plausible.Imported.load_import_data(site)- merge_assigns(conn, site: site, current_user_role: role, current_user_id: user_id)+ merge_assigns(conn, site: site, current_user_role: role)elsePlausibleWeb.ControllerHelpers.render_error(conn, 404) |> haltenddiff --git a/lib/plausible_web/plugs/inject_debug_headers.ex b/lib/plausible_web/plugs/inject_debug_headers.exdeleted file mode 100644index b0f3cf77ba..0000000000--- a/lib/plausible_web/plugs/inject_debug_headers.ex+++ /dev/null@@ -1,25 +0,0 @@-defmodule PlausibleWeb.Plugs.InjectDebugHeaders do- @moduledoc """- This plug updates the response with debug query headers,- granted they were tracked via `Plausible.DebugReplayInfo`.- """-- def init(opts), do: opts-- def call(conn, _opts \\ []) do- Plug.Conn.register_before_send(conn, fn conn ->- Plausible.DebugReplayInfo.get_queries_from_context()- |> Enum.reverse()- |> Enum.with_index()- |> Enum.reduce(conn, fn {q, index}, conn ->- {[label], [value]} = Enum.unzip(q)-- conn- |> Plug.Conn.put_resp_header(- "x-plausible-query-#{String.pad_leading("#{index}", 3, "0")}-#{label}",- String.replace(value, ["\n", "\r", "\x00"], "")- )- end)- end)- end-enddiff --git a/test/plausible/clickhouse_repo_test.exs b/test/plausible/clickhouse_repo_test.exsdeleted file mode 100644index 841bcd1365..0000000000--- a/test/plausible/clickhouse_repo_test.exs+++ /dev/null@@ -1,110 +0,0 @@-defmodule Plausible.ClickhouseRepoTest do- use Plausible.DataCase, async: true-- alias Plausible.ClickhouseRepo- alias Plausible.DebugReplayInfo-- import ExUnit.CaptureLog-- test "queries are kept in context for debugging purposes for superadmins" do- Sentry.Context.set_user_context(%{id: 1, super_admin?: true})-- ClickhouseRepo.all(from(u in "events_v2", select: true, limit: 0))-- ClickhouseRepo.one(from(u in "events_v2", select: true, limit: 0),- label: "one"- )-- queries = DebugReplayInfo.get_queries_from_context()-- assert [- %{"one" => "SELECT true FROM \"events_v2\" AS e0 LIMIT 0"},- %{"unlabelled" => "SELECT true FROM \"events_v2\" AS e0 LIMIT 0"}- ] = queries- end-- test "queries are not kept in context for debugging purposes for non-superadmins" do- ClickhouseRepo.all(from(u in "events_v2", select: true, limit: 0))-- ClickhouseRepo.one(from(u in "events_v2", select: true, limit: 0),- label: "one"- )-- assert DebugReplayInfo.get_queries_from_context() == []- end-- test "queries are logged with sentry and extra metadata" do- Sentry.Context.set_user_context(%{id: 1})- Sentry.Context.set_request_context(%{url: "http://example.com"})- Sentry.Context.set_extra_context(%{domain: "example.com", site_id: 1})-- ClickhouseRepo.all(from(u in "sessions_v2", select: true, limit: 0),- label: "log_all"- )-- ClickhouseRepo.one(from(u in "sessions_v2", select: true, limit: 0),- label: "log_one",- metadata: %{"metric" => "value"}- )-- flush_clickhouse_logs()-- assert %{- rows: [- [q1, c1],- [q2, c2]- ]- } =- ClickhouseRepo.query!("""- SELECT- query,- log_comment- FROM system.query_log- WHERE (type = 1) AND (query LIKE '%sessions_v2%')- AND JSONExtractString(log_comment, 'label') IN ('log_all', 'log_one')- ORDER BY event_time DESC- LIMIT 2- """)-- assert q1 == "SELECT true FROM \"sessions_v2\" AS s0 LIMIT 0"- assert q1 == q2-- assert Enum.find([c1, c2], fn c ->- Jason.decode!(c) == %{- "label" => "log_all",- "domain" => "example.com",- "url" => "http://example.com",- "user_id" => 1,- "metadata" => %{},- "site_id" => 1- }- end)-- assert Enum.find([c1, c2], fn c ->- Jason.decode!(c) == %{- "label" => "log_one",- "domain" => "example.com",- "url" => "http://example.com",- "user_id" => 1,- "metadata" => %{"metric" => "value"},- "site_id" => 1- }- end)- end-- test "non-serializable log comment won't cause the query to crash and will log an error" do- log =- capture_log(fn ->- assert [] =- ClickhouseRepo.all(from(u in "sessions_v2", select: true, limit: 0),- label: "log_all",- metadata: {:error, :skip_me}- )- end)-- assert log =~- ~s/Failed to include log comment: %{label: "log_all", domain: nil, metadata: {:error, :skip_me}, url: nil, site_id: nil, user_id: nil}/- end-- defp flush_clickhouse_logs(), do: Plausible.IngestRepo.query!("SYSTEM FLUSH LOGS")-enddiff --git a/test/plausible/debug_replay_info_test.exs b/test/plausible/debug_replay_info_test.exsindex 89c6ce6cab..b0379aa155 100644--- a/test/plausible/debug_replay_info_test.exs+++ b/test/plausible/debug_replay_info_test.exs@@ -46,31 +46,4 @@ defmodule Plausible.DebugReplayInfoTest doassert context.extra.debug_replay_info == :too_largeassert context.extra.debug_replay_info_size > 10_000end-- describe "query tracking" do- test "track and retrieve queries" do- :ok = Plausible.DebugReplayInfo.track_query("SELECT * FROM users", "users")- :ok = Plausible.DebugReplayInfo.track_query("SELECT * FROM accounts", "accounts")-- assert [%{"accounts" => "SELECT * FROM accounts"}, %{"users" => "SELECT * FROM users"}] =- Plausible.DebugReplayInfo.get_queries_from_context()- end-- test "carry over context" do- Sentry.Context.set_user_context(%{id: 1})- Sentry.Context.set_request_context(%{url: "http://example.com"})- Sentry.Context.set_extra_context(%{domain: "example.com", site_id: 1})-- test = self()-- sentry_ctx = Sentry.Context.get_all()-- Task.start(fn ->- Plausible.DebugReplayInfo.carry_over_context(sentry_ctx)- send(test, {:task_context, Sentry.Context.get_all()})- end)-- assert_receive {:task_context, ^sentry_ctx}- end- endenddiff --git a/test/plausible_web/plugs/inject_debug_headers_test.exs b/test/plausible_web/plugs/inject_debug_headers_test.exsdeleted file mode 100644index b8db4556f4..0000000000--- a/test/plausible_web/plugs/inject_debug_headers_test.exs+++ /dev/null@@ -1,39 +0,0 @@-defmodule PlausibleWeb.Plugs.InjectDebugHeadersTest do- use Plausible.DataCase, async: true- use Plug.Test-- alias PlausibleWeb.Plugs.InjectDebugHeaders- @prefix "x-plausible-"-- test "does not inject #{@prefix} headers when no queries are registered" do- conn = :get |> conn("/") |> InjectDebugHeaders.call() |> send_resp(200, "")- assert Enum.filter(conn.resp_headers, fn {k, _} -> String.starts_with?(k, @prefix) end) == []- end-- test "injects #{@prefix} headers when queries are registered" do- :ok = Plausible.DebugReplayInfo.track_query("SELECT * FROM users", "users")- :ok = Plausible.DebugReplayInfo.track_query("SELECT * FROM accounts", "accounts")-- conn = :get |> conn("/") |> InjectDebugHeaders.call() |> send_resp(200, "")-- assert Enum.filter(conn.resp_headers, fn {k, _} -> String.starts_with?(k, @prefix) end) ==- [- {"x-plausible-query-000-users", "SELECT * FROM users"},- {"x-plausible-query-001-accounts", "SELECT * FROM accounts"}- ]- end-- test "skips invalid header chars" do- :ok = Plausible.DebugReplayInfo.track_query("\nfoo", "trap1")- :ok = Plausible.DebugReplayInfo.track_query("\rbar", "trap2")- :ok = Plausible.DebugReplayInfo.track_query("\x00baz", "trap3")- conn = :get |> conn("/") |> InjectDebugHeaders.call() |> send_resp(200, "")-- assert Enum.filter(conn.resp_headers, fn {k, _} -> String.starts_with?(k, @prefix) end) ==- [- {"x-plausible-query-000-trap1", "foo"},- {"x-plausible-query-001-trap2", "bar"},- {"x-plausible-query-002-trap3", "baz"}- ]- end-end