Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Debug queries for super-admins (#4010)
hq1dd493fdad21f
diff --git a/assets/js/dashboard/api.js b/assets/js/dashboard/api.jsindex 26fd29916e..a14c938f96 100644--- a/assets/js/dashboard/api.js+++ b/assets/js/dashboard/api.js@@ -61,9 +61,10 @@ export function serializeQuery(query, extraQuery = []) {export function get(url, query = {}, ...extraQuery) {const headers = SHARED_LINK_AUTH ? { 'X-Shared-Link-Auth': SHARED_LINK_AUTH } : {}- url = url + serializeQuery(query, extraQuery)- return fetch(url, { signal: abortController.signal, headers: headers })+ const serializedUrl = url + serializeQuery(query, extraQuery)+ return fetch(serializedUrl, { signal: abortController.signal, headers: headers }).then(response => {+ logDebugHeaders(url, response.headers)if (!response.ok) {return response.json().then((msg) => {throw new ApiError(msg.error, msg)@@ -73,6 +74,13 @@ export function get(url, query = {}, ...extraQuery) {})}+function logDebugHeaders(url, headers) {+ const debugHeaders = Array.from(headers).filter(([h]) => h.startsWith("x-plausible"))+ if (debugHeaders.length > 0) {+ console.info(url, Object.fromEntries(debugHeaders))+ }+}+export function put(url, body) {return fetch(url, {method: "PUT",diff --git a/extra/lib/plausible/stats/funnel.ex b/extra/lib/plausible/stats/funnel.exindex d9c497cee6..42248f5e18 100644--- a/extra/lib/plausible/stats/funnel.ex+++ b/extra/lib/plausible/stats/funnel.ex@@ -74,7 +74,7 @@ defmodule Plausible.Stats.Funnel dogroup_by: f.step)- ClickhouseRepo.all(query)+ ClickhouseRepo.all(query, label: :query_funnel)enddefp select_funnel(db_query, funnel_definition) dodiff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.exindex f6f8631d97..05b4184635 100644--- a/lib/plausible/clickhouse_repo.ex+++ b/lib/plausible/clickhouse_repo.ex@@ -4,6 +4,8 @@ defmodule Plausible.ClickhouseRepo doadapter: Ecto.Adapters.ClickHouse,read_only: true+ require Logger+defmacro __using__(_) doquote doalias Plausible.ClickhouseRepo@@ -12,17 +14,69 @@ defmodule Plausible.ClickhouseRepo doendend+ @impl Ecto.Repo+ def prepare_query(operation, query, opts) do+ if Plausible.DebugReplayInfo.super_admin?() do+ Plausible.DebugReplayInfo.track_query(+ to_inline_sql(operation, query),+ opts[:label] || "unlabelled"+ )+ end++ {query, include_log_comment(opts)}+ end+@task_timeout 60_000def parallel_tasks(queries) do- ctx = OpenTelemetry.Ctx.get_current()+ otel_ctx = OpenTelemetry.Ctx.get_current()+ sentry_ctx = Sentry.Context.get_all()execute_with_tracing = fn fun ->- OpenTelemetry.Ctx.attach(ctx)- fun.()+ Plausible.DebugReplayInfo.carry_over_context(sentry_ctx)+ OpenTelemetry.Ctx.attach(otel_ctx)+ result = fun.()+ {Sentry.Context.get_all(), result}endTask.async_stream(queries, execute_with_tracing, max_concurrency: 3, timeout: @task_timeout)|> Enum.to_list()|> Keyword.values()+ |> Enum.map(fn {sentry_ctx, result} ->+ set_sentry_context(sentry_ctx)+ result+ end)+ end++ defp set_sentry_context(previous_sentry_ctx) do+ Plausible.DebugReplayInfo.carry_over_context(previous_sentry_ctx)+ previous_queries = Plausible.DebugReplayInfo.get_queries_from_context(previous_sentry_ctx)+ current_queries = Plausible.DebugReplayInfo.get_queries_from_context()++ Sentry.Context.set_extra_context(%{+ queries: previous_queries ++ current_queries+ })+ end++ defp include_log_comment(opts) do+ sentry_context = Sentry.Context.get_all()++ log_comment = %{+ user_id: sentry_context[:user][:id],+ label: opts[:label] || "unlabelled",+ url: sentry_context[:request][:url],+ domain: sentry_context[:extra][:domain],+ site_id: sentry_context[:extra][:site_id],+ metadata: opts[:metadata] || %{}+ }++ case Jason.encode(log_comment) do+ {:ok, encoded} ->+ setting = {:log_comment, encoded}+ Keyword.update(opts, :settings, [setting], fn settings -> [setting | settings] end)++ {:error, _} ->+ Logger.error("Failed to include log comment: #{inspect(log_comment)}")+ opts+ endendenddiff --git a/lib/plausible/debug_replay_info.ex b/lib/plausible/debug_replay_info.exindex ed1b90fbf5..8d418f65bb 100644--- a/lib/plausible/debug_replay_info.ex+++ b/lib/plausible/debug_replay_info.ex@@ -42,6 +42,40 @@ defmodule Plausible.DebugReplayInfo doendend+ def super_admin? do+ context = Sentry.Context.get_all()++ case context[:user] do+ %{super_admin?: true} ->+ true++ _ ->+ false+ end+ end++ def track_query(query, label) do+ queries = get_queries_from_context()++ Sentry.Context.set_extra_context(%{+ queries: [%{label => query} | queries]+ })+ end++ def carry_over_context(sentry_ctx) do+ Sentry.Context.set_user_context(sentry_ctx[:user])+ Sentry.Context.set_request_context(sentry_ctx[:request])++ Sentry.Context.set_extra_context(%{+ domain: sentry_ctx[:extra][:domain],+ site_id: sentry_ctx[:extra][:site_id]+ })+ end++ def get_queries_from_context(context \\ Sentry.Context.get_all()) do+ get_in(context, [:extra, :queries]) || []+ end+@spec deserialize(String.t()) :: any()def deserialize(replay_info) doreplay_infodiff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 27d4290b01..dcf8d3b88e 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -46,7 +46,7 @@ defmodule Plausible.Stats.Aggregate dofrom(e in base_event_query(site, query), select: ^select_event_metrics(metrics))|> merge_imported(site, query, :aggregate, metrics)|> maybe_add_conversion_rate(site, query, metrics, include_imported: query.include_imported)- |> ClickhouseRepo.one()+ |> ClickhouseRepo.one(label: :aggregate_events)enddefp aggregate_sessions(_, _, []), do: %{}@@ -55,7 +55,7 @@ defmodule Plausible.Stats.Aggregate dofrom(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))|> filter_converted_sessions(site, query)|> merge_imported(site, query, :aggregate, metrics)- |> ClickhouseRepo.one()+ |> ClickhouseRepo.one(label: :aggregate_sessions)|> Util.keep_requested_metrics(metrics)end@@ -187,7 +187,7 @@ defmodule Plausible.Stats.Aggregate dofrom e in Ecto.Query.subquery(avg_time_per_page_transition_q),select: fragment("avg(ifNotFinite(?,NULL))", e.avg)- %{time_on_page: ClickhouseRepo.one(time_on_page_q)}+ %{time_on_page: ClickhouseRepo.one(time_on_page_q, label: :time_on_page)}end@metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent]diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex e2b0b40667..e0bb1c871b 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -111,7 +111,7 @@ defmodule Plausible.Stats.Breakdown doif full_q dofull_q|> maybe_add_conversion_rate(site, query, metrics, include_imported: false)- |> ClickhouseRepo.all()+ |> ClickhouseRepo.all(label: :breakdown_by_goal)|> transform_keys(%{name: :goal})|> cast_revenue_metrics_to_money(revenue_goals)|> Util.keep_requested_metrics(metrics)@@ -309,7 +309,7 @@ defmodule Plausible.Stats.Breakdown dodefp paginate_and_execute(q, metrics, pagination) doq|> apply_pagination(pagination)- |> ClickhouseRepo.all()+ |> ClickhouseRepo.all(label: :paginate_and_execute)|> Util.keep_requested_metrics(metrics)end@@ -405,7 +405,7 @@ defmodule Plausible.Stats.Breakdown doendtimed_pages_q- |> Plausible.ClickhouseRepo.all()+ |> Plausible.ClickhouseRepo.all(label: :breakdown_time_on_page)|> Map.new()enddiff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex 52d2e889ce..6c90bcf0ad 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -46,7 +46,7 @@ defmodule Plausible.Stats.Clickhouse dogroup_by: i.import_id,select: {i.import_id, sum(i.pageviews)})- |> Plausible.ClickhouseRepo.all()+ |> Plausible.ClickhouseRepo.all(label: :imported_pageview_counts)|> Map.new()end@@ -117,7 +117,7 @@ defmodule Plausible.Stats.Clickhouse doreferrers = Plausible.Stats.Sampling.add_query_hint(referrers, 10_000_000)end- ClickhouseRepo.all(referrers)+ ClickhouseRepo.all(referrers, label: :referrers)enddef current_visitors(site, query) do@@ -168,7 +168,7 @@ defmodule Plausible.Stats.Clickhouse doprevious_result =previous_query- |> ClickhouseRepo.all()+ |> ClickhouseRepo.all(label: :last_24h_visitors_previous)|> Enum.reduce(%{}, fn%{total_visitors: total, site_id: site_id}, acc -> Map.put_new(acc, site_id, total)end)@@ -199,7 +199,7 @@ defmodule Plausible.Stats.Clickhouse doresult =current_q- |> ClickhouseRepo.all()+ |> ClickhouseRepo.all(label: :last_24h_visitors_current)|> Enum.group_by(& &1.site_id)|> Enum.map(fn {site_id, entries} ->%{total: visitors} = List.first(entries)diff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.exindex 8fe69b9d44..c2d8ad7716 100644--- a/lib/plausible/stats/custom_props.ex+++ b/lib/plausible/stats/custom_props.ex@@ -19,7 +19,7 @@ defmodule Plausible.Stats.CustomProps dodistinct: true)|> maybe_allowed_props_only(site)- |> ClickhouseRepo.all()+ |> ClickhouseRepo.all(label: :fetch_prop_names)endenddiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 3b400907b1..6908e2761c 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -58,7 +58,7 @@ defmodule Plausible.Stats.Timeseries do|> select_bucket(site, query)|> maybe_add_timeseries_conversion_rate(site, query, metrics)|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)- |> ClickhouseRepo.all()+ |> ClickhouseRepo.all(label: :event_timeseries)enddefp sessions_timeseries(_, _, []), do: []@@ -68,7 +68,7 @@ defmodule Plausible.Stats.Timeseries do|> filter_converted_sessions(site, query)|> select_bucket(site, query)|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)- |> ClickhouseRepo.all()+ |> ClickhouseRepo.all(label: :session_timeseries)|> Util.keep_requested_metrics(metrics)enddiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 8b411eef0b..f85765941e 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -12,7 +12,8 @@ defmodule PlausibleWeb.Api.StatsController do@revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])- plug(:date_validation_plug)+ plug :date_validation_plug+ plug PlausibleWeb.Plugs.InjectDebugHeaders@doc """Returns a time-series based on given parameters.diff --git a/lib/plausible_web/plugs/authorize_site_access.ex b/lib/plausible_web/plugs/authorize_site_access.exindex de35461c50..3306a5229e 100644--- a/lib/plausible_web/plugs/authorize_site_access.ex+++ b/lib/plausible_web/plugs/authorize_site_access.ex@@ -20,14 +20,17 @@ defmodule PlausibleWeb.AuthorizeSiteAccess doPlausibleWeb.ControllerHelpers.render_error(conn, 404) |> haltelseuser_id = get_session(conn, :current_user_id)+membership_role = user_id && Plausible.Sites.role(user_id, site)+ super_admin? = user_id && Plausible.Auth.is_super_admin?(user_id)+role =cond douser_id && membership_role ->membership_role- Plausible.Auth.is_super_admin?(user_id) ->+ super_admin? ->:super_adminsite.public ->@@ -41,7 +44,8 @@ defmodule PlausibleWeb.AuthorizeSiteAccess doendif role in allowed_roles do- Sentry.Context.set_user_context(%{id: user_id})+ Sentry.Context.set_user_context(%{id: user_id, super_admin?: super_admin?})+Plausible.OpenTelemetry.add_user_attributes(user_id)Sentry.Context.set_extra_context(%{site_id: site.id, domain: site.domain})@@ -49,7 +53,7 @@ defmodule PlausibleWeb.AuthorizeSiteAccess dosite = Plausible.Imported.load_import_data(site)- merge_assigns(conn, site: site, current_user_role: role)+ merge_assigns(conn, site: site, current_user_role: role, current_user_id: user_id)elsePlausibleWeb.ControllerHelpers.render_error(conn, 404) |> haltenddiff --git a/lib/plausible_web/plugs/inject_debug_headers.ex b/lib/plausible_web/plugs/inject_debug_headers.exnew file mode 100644index 0000000000..b0f3cf77ba--- /dev/null+++ b/lib/plausible_web/plugs/inject_debug_headers.ex@@ -0,0 +1,25 @@+defmodule PlausibleWeb.Plugs.InjectDebugHeaders do+ @moduledoc """+ This plug updates the response with debug query headers,+ granted they were tracked via `Plausible.DebugReplayInfo`.+ """++ def init(opts), do: opts++ def call(conn, _opts \\ []) do+ Plug.Conn.register_before_send(conn, fn conn ->+ Plausible.DebugReplayInfo.get_queries_from_context()+ |> Enum.reverse()+ |> Enum.with_index()+ |> Enum.reduce(conn, fn {q, index}, conn ->+ {[label], [value]} = Enum.unzip(q)++ conn+ |> Plug.Conn.put_resp_header(+ "x-plausible-query-#{String.pad_leading("#{index}", 3, "0")}-#{label}",+ String.replace(value, ["\n", "\r", "\x00"], "")+ )+ end)+ end)+ end+enddiff --git a/mix.exs b/mix.exsindex 2166808efa..f839133bf4 100644--- a/mix.exs+++ b/mix.exs@@ -69,7 +69,7 @@ defmodule Plausible.MixProject do{:bamboo_mua, "~> 0.1.4"},{:bcrypt_elixir, "~> 3.0"},{:bypass, "~> 2.1", only: [:dev, :test, :ce_test]},- {:ecto_ch, "~> 0.3"},+ {:ecto_ch, github: "plausible/ecto_ch", branch: "pretty-sql"},{:cloak, "~> 1.1"},{:cloak_ecto, "~> 1.2"},{:combination, "~> 0.0.3"},diff --git a/mix.lock b/mix.lockindex 240acdbc55..5918ee3859 100644--- a/mix.lock+++ b/mix.lock@@ -8,7 +8,7 @@"bcrypt_elixir": {:hex, :bcrypt_elixir, "3.1.0", "0b110a9a6c619b19a7f73fa3004aa11d6e719a67e672d1633dc36b6b2290a0f7", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "2ad2acb5a8bc049e8d5aa267802631912bb80d5f4110a178ae7999e69dca1bf7"},"bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},"bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"},- "castore": {:hex, :castore, "1.0.5", "9eeebb394cc9a0f3ae56b813459f990abb0a3dedee1be6b27fdb50301930502f", [:mix], [], "hexpm", "8d7c597c3e4a64c395980882d4bca3cebb8d74197c590dc272cfd3b6a6310578"},+ "castore": {:hex, :castore, "1.0.6", "ffc42f110ebfdafab0ea159cd43d31365fa0af0ce4a02ecebf1707ae619ee727", [:mix], [], "hexpm", "374c6e7ca752296be3d6780a6d5b922854ffcc74123da90f2f328996b962d33a"},"certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"},"ch": {:hex, :ch, "0.2.5", "b8d70689951bd14c8c8791dc72cdc957ba489ceae723e79cf1a91d95b6b855ae", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "97de104c8f513a23c6d673da37741f68ae743f6cdb654b96a728d382e2fba4de"},"chatterbox": {:hex, :ts_chatterbox, "0.15.1", "5cac4d15dd7ad61fc3c4415ce4826fc563d4643dee897a558ec4ea0b1c835c9c", [:rebar3], [{:hpack, "~> 0.3.0", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "4f75b91451338bc0da5f52f3480fa6ef6e3a2aeecfc33686d6b3d0a0948f31aa"},@@ -32,8 +32,8 @@"digital_token": {:hex, :digital_token, "0.6.0", "13e6de581f0b1f6c686f7c7d12ab11a84a7b22fa79adeb4b50eec1a2d278d258", [:mix], [{:cldr_utils, "~> 2.17", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "2455d626e7c61a128b02a4a8caddb092548c3eb613ac6f6a85e4cbb6caddc4d1"},"double": {:hex, :double, "0.8.2", "8e1cfcccdaef76c18846bc08e555555a2a699b806fa207b6468572a60513cc6a", [:mix], [], "hexpm", "90287642b2ec86125e0457aaba2ab0e80f7d7050cc80a0cef733e59bd70aa67c"},"earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"},- "ecto": {:hex, :ecto, "3.11.1", "4b4972b717e7ca83d30121b12998f5fcdc62ba0ed4f20fd390f16f3270d85c3e", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebd3d3772cd0dfcd8d772659e41ed527c28b2a8bde4b00fe03e0463da0f1983b"},- "ecto_ch": {:hex, :ecto_ch, "0.3.2", "b6e7d0a6ad412662d7727ba1b5128a1a30a0835e1b4168db79e5b2551a27ba50", [:mix], [{:ch, "~> 0.2.0", [hex: :ch, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.9", [hex: :ecto_sql, repo: "hexpm", optional: false]}], "hexpm", "a195d0f09d1177d9b11f00ef608895db1e02711d0a18730e98fbb50055758e6d"},+ "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"},+ "ecto_ch": {:git, "https://github.com/plausible/ecto_ch.git", "6a3a74364f514810c4f073575a33ab3be891f560", [branch: "pretty-sql"]},"ecto_network": {:hex, :ecto_network, "1.5.0", "a930c910975e7a91237b858ebf0f4ad7b2aae32fa846275aa203cb858459ec73", [:mix], [{:ecto_sql, ">= 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:phoenix_html, ">= 0.0.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.14.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "4d614434ae3e6d373a2f693d56aafaa3f3349714668ffd6d24e760caf578aa2f"},"ecto_sql": {:hex, :ecto_sql, "3.11.1", "e9abf28ae27ef3916b43545f9578b4750956ccea444853606472089e7d169470", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ce14063ab3514424276e7e360108ad6c2308f6d88164a076aac8a387e1fea634"},"elixir_make": {:hex, :elixir_make, "0.7.7", "7128c60c2476019ed978210c245badf08b03dbec4f24d05790ef791da11aa17c", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5bc19fff950fad52bbe5f211b12db9ec82c6b34a9647da0c2224b8b8464c7e6c"},@@ -121,7 +121,7 @@"plug": {:hex, :plug, "1.15.3", "712976f504418f6dff0a3e554c40d705a9bcf89a7ccef92fc6a5ef8f16a30a97", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4365a3c010a56af402e0809208873d113e9c38c401cabd88027ef4f5c01fd2"},"plug_cowboy": {:hex, :plug_cowboy, "2.7.0", "3ae9369c60641084363b08fe90267cbdd316df57e3557ea522114b30b63256ea", [:mix], [{:cowboy, "~> 2.7.0 or ~> 2.8.0 or ~> 2.9.0 or ~> 2.10.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d85444fb8aa1f2fc62eabe83bbe387d81510d773886774ebdcb429b3da3c1a4a"},"plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"},- "postgrex": {:hex, :postgrex, "0.17.4", "5777781f80f53b7c431a001c8dad83ee167bcebcf3a793e3906efff680ab62b3", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "6458f7d5b70652bc81c3ea759f91736c16a31be000f306d3c64bcdfe9a18b3cc"},+ "postgrex": {:hex, :postgrex, "0.17.5", "0483d054938a8dc069b21bdd636bf56c487404c241ce6c319c1f43588246b281", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "50b8b11afbb2c4095a3ba675b4f055c416d0f3d7de6633a595fc131a828a67eb"},"prom_ex": {:hex, :prom_ex, "1.9.0", "63e6dda6c05cdeec1f26c48443dcc38ffd2118b3665ae8d2bd0e5b79f2aea03e", [:mix], [{:absinthe, ">= 1.6.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.0.2", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.5.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.15", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.4.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.3", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.5.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.14.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.12.1", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.5", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "01f3d4f69ec93068219e686cc65e58a29c42bea5429a8ff4e2121f19db178ee6"},"public_suffix": {:git, "https://github.com/axelson/publicsuffix-elixir", "fa40c243d4b5d8598b90cff268bc4e33f3bb63f1", []},"ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},diff --git a/test/plausible/clickhouse_repo_test.exs b/test/plausible/clickhouse_repo_test.exsnew file mode 100644index 0000000000..841bcd1365--- /dev/null+++ b/test/plausible/clickhouse_repo_test.exs@@ -0,0 +1,110 @@+defmodule Plausible.ClickhouseRepoTest do+ use Plausible.DataCase, async: true++ alias Plausible.ClickhouseRepo+ alias Plausible.DebugReplayInfo++ import ExUnit.CaptureLog++ test "queries are kept in context for debugging purposes for superadmins" do+ Sentry.Context.set_user_context(%{id: 1, super_admin?: true})++ ClickhouseRepo.all(from(u in "events_v2", select: true, limit: 0))++ ClickhouseRepo.one(from(u in "events_v2", select: true, limit: 0),+ label: "one"+ )++ queries = DebugReplayInfo.get_queries_from_context()++ assert [+ %{"one" => "SELECT true FROM \"events_v2\" AS e0 LIMIT 0"},+ %{"unlabelled" => "SELECT true FROM \"events_v2\" AS e0 LIMIT 0"}+ ] = queries+ end++ test "queries are not kept in context for debugging purposes for non-superadmins" do+ ClickhouseRepo.all(from(u in "events_v2", select: true, limit: 0))++ ClickhouseRepo.one(from(u in "events_v2", select: true, limit: 0),+ label: "one"+ )++ assert DebugReplayInfo.get_queries_from_context() == []+ end++ test "queries are logged with sentry and extra metadata" do+ Sentry.Context.set_user_context(%{id: 1})+ Sentry.Context.set_request_context(%{url: "http://example.com"})+ Sentry.Context.set_extra_context(%{domain: "example.com", site_id: 1})++ ClickhouseRepo.all(from(u in "sessions_v2", select: true, limit: 0),+ label: "log_all"+ )++ ClickhouseRepo.one(from(u in "sessions_v2", select: true, limit: 0),+ label: "log_one",+ metadata: %{"metric" => "value"}+ )++ flush_clickhouse_logs()++ assert %{+ rows: [+ [q1, c1],+ [q2, c2]+ ]+ } =+ ClickhouseRepo.query!("""+ SELECT+ query,+ log_comment+ FROM system.query_log+ WHERE (type = 1) AND (query LIKE '%sessions_v2%')+ AND JSONExtractString(log_comment, 'label') IN ('log_all', 'log_one')+ ORDER BY event_time DESC+ LIMIT 2+ """)++ assert q1 == "SELECT true FROM \"sessions_v2\" AS s0 LIMIT 0"+ assert q1 == q2++ assert Enum.find([c1, c2], fn c ->+ Jason.decode!(c) == %{+ "label" => "log_all",+ "domain" => "example.com",+ "url" => "http://example.com",+ "user_id" => 1,+ "metadata" => %{},+ "site_id" => 1+ }+ end)++ assert Enum.find([c1, c2], fn c ->+ Jason.decode!(c) == %{+ "label" => "log_one",+ "domain" => "example.com",+ "url" => "http://example.com",+ "user_id" => 1,+ "metadata" => %{"metric" => "value"},+ "site_id" => 1+ }+ end)+ end++ test "non-serializable log comment won't cause the query to crash and will log an error" do+ log =+ capture_log(fn ->+ assert [] =+ ClickhouseRepo.all(from(u in "sessions_v2", select: true, limit: 0),+ label: "log_all",+ metadata: {:error, :skip_me}+ )+ end)++ assert log =~+ ~s/Failed to include log comment: %{label: "log_all", domain: nil, metadata: {:error, :skip_me}, url: nil, site_id: nil, user_id: nil}/+ end++ defp flush_clickhouse_logs(), do: Plausible.IngestRepo.query!("SYSTEM FLUSH LOGS")+enddiff --git a/test/plausible/debug_replay_info_test.exs b/test/plausible/debug_replay_info_test.exsindex b0379aa155..89c6ce6cab 100644--- a/test/plausible/debug_replay_info_test.exs+++ b/test/plausible/debug_replay_info_test.exs@@ -46,4 +46,31 @@ defmodule Plausible.DebugReplayInfoTest doassert context.extra.debug_replay_info == :too_largeassert context.extra.debug_replay_info_size > 10_000end++ describe "query tracking" do+ test "track and retrieve queries" do+ :ok = Plausible.DebugReplayInfo.track_query("SELECT * FROM users", "users")+ :ok = Plausible.DebugReplayInfo.track_query("SELECT * FROM accounts", "accounts")++ assert [%{"accounts" => "SELECT * FROM accounts"}, %{"users" => "SELECT * FROM users"}] =+ Plausible.DebugReplayInfo.get_queries_from_context()+ end++ test "carry over context" do+ Sentry.Context.set_user_context(%{id: 1})+ Sentry.Context.set_request_context(%{url: "http://example.com"})+ Sentry.Context.set_extra_context(%{domain: "example.com", site_id: 1})++ test = self()++ sentry_ctx = Sentry.Context.get_all()++ Task.start(fn ->+ Plausible.DebugReplayInfo.carry_over_context(sentry_ctx)+ send(test, {:task_context, Sentry.Context.get_all()})+ end)++ assert_receive {:task_context, ^sentry_ctx}+ end+ endenddiff --git a/test/plausible_web/plugs/inject_debug_headers_test.exs b/test/plausible_web/plugs/inject_debug_headers_test.exsnew file mode 100644index 0000000000..b8db4556f4--- /dev/null+++ b/test/plausible_web/plugs/inject_debug_headers_test.exs@@ -0,0 +1,39 @@+defmodule PlausibleWeb.Plugs.InjectDebugHeadersTest do+ use Plausible.DataCase, async: true+ use Plug.Test++ alias PlausibleWeb.Plugs.InjectDebugHeaders+ @prefix "x-plausible-"++ test "does not inject #{@prefix} headers when no queries are registered" do+ conn = :get |> conn("/") |> InjectDebugHeaders.call() |> send_resp(200, "")+ assert Enum.filter(conn.resp_headers, fn {k, _} -> String.starts_with?(k, @prefix) end) == []+ end++ test "injects #{@prefix} headers when queries are registered" do+ :ok = Plausible.DebugReplayInfo.track_query("SELECT * FROM users", "users")+ :ok = Plausible.DebugReplayInfo.track_query("SELECT * FROM accounts", "accounts")++ conn = :get |> conn("/") |> InjectDebugHeaders.call() |> send_resp(200, "")++ assert Enum.filter(conn.resp_headers, fn {k, _} -> String.starts_with?(k, @prefix) end) ==+ [+ {"x-plausible-query-000-users", "SELECT * FROM users"},+ {"x-plausible-query-001-accounts", "SELECT * FROM accounts"}+ ]+ end++ test "skips invalid header chars" do+ :ok = Plausible.DebugReplayInfo.track_query("\nfoo", "trap1")+ :ok = Plausible.DebugReplayInfo.track_query("\rbar", "trap2")+ :ok = Plausible.DebugReplayInfo.track_query("\x00baz", "trap3")+ conn = :get |> conn("/") |> InjectDebugHeaders.call() |> send_resp(200, "")++ assert Enum.filter(conn.resp_headers, fn {k, _} -> String.starts_with?(k, @prefix) end) ==+ [+ {"x-plausible-query-000-trap1", "foo"},+ {"x-plausible-query-001-trap2", "bar"},+ {"x-plausible-query-002-trap3", "baz"}+ ]+ end+end